diff options
107 files changed, 2032 insertions, 847 deletions
diff --git a/README-documentation.md b/README-documentation.md index 81d5d93199e..01a95ba6ad4 100644 --- a/README-documentation.md +++ b/README-documentation.md @@ -62,6 +62,10 @@ Make the text as short, clear, and easy to read as possible: * Avoid superfluous words such as "very". * Avoid filler sentences intended to improve the flow of the text - documents are usually browsed, not read anyway. * Use consistent terminology even when it leads to repetition which would be bad in other kind of writing. +* Use active form "index the documents", not passive "indexing the documents" +* Avoid making it personal - do not use "we", "you", "our" +* Do not use &quot; , &mdash; and the likes - makes the document harder to edit, and no need to use it. +* Less is more - <em> and <strong> is sufficient formatting in most cases ### Links @@ -72,4 +76,4 @@ Example: <h2 id="my-nice-heading">My nice Heading</h2> If this algorithmic transformation is followed it is possible to link to this section using <a href="doc.html#my-nice-heading"> without having to consult the html source of the page to find the right id. -*By Jon Bratseth in June 2016*
\ No newline at end of file +*By Jon Bratseth in June 2016* diff --git a/README.md b/README.md index c8d3f69ccd3..c1ab45629f0 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,8 @@ data can be performed at serving time. This README describes how to build and develop the Vespa engine. If you want to use Vespa you can go to the -[quick start guide](http://yahoo.github.io/vespa/vespa-quick-start.html), or take a -look at our -[user documentation](http://yahoo.github.io/vespa/vespatoc.html). +[quick start guide](http://yahoo.github.io/vespa/vespa-quick-start.html), or find the full +documentation and other resources at http://yahoo.github.io/vespa/. ## Getting started developing @@ -19,8 +18,20 @@ look at our ### Setting up build environment - sudo yum -y install epel-release - # TODO: Install build deps or depend on Build-Require in .spec file? +C++ building is supported on CentOS 7. + +#### Install required build dependencies + + sudo yum -y install epel-release centos-release-scl yum-utils + sudo yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo + sudo yum -y install devtoolset-4-gcc-c++ devtoolset-4-libatomic-devel \ + Judy-devel cmake3 ccache lz4-devel zlib-devel maven libicu-devel llvm-devel \ + llvm-static java-1.8.0-openjdk-devel openssl-devel rpm-build make \ + vespa-boost-devel vespa-libtorrent-devel vespa-zookeeper-c-client-devel vespa-cppunit-devel + +or use our prebuilt docker image + + # TODO: Add docker command ### Building Java modules @@ -31,11 +42,8 @@ Java modules can be built on any environment having Java and Maven: ### Building C++ modules -C++ building is currently supported on CentOS 7: - -TODO: List required build dependencies - - sh bootstrap.sh + source /opt/rh/devtoolset-4/enable + sh bootstrap.sh full cmake . make make test diff --git a/bootstrap.sh b/bootstrap.sh index 60145b39077..66b399fac13 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,6 +1,16 @@ #!/bin/bash -e # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +FULL=false + +if [ "$1" = "full" ]; then + FULL=true +fi + +mvn_install() { + mvn install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true $@ +} + # Generate vtag map top=$(dirname $0) $top/dist/getversion.pl -M $top > $top/dist/vtag.map @@ -9,15 +19,20 @@ $top/dist/getversion.pl -M $top > $top/dist/vtag.map # Vespa java dependency graph. MODULES=" parent - configgen annotations scalalib bundle-plugin - config-class-plugin - yolean - vespajlib - filedistributionmanager" + " for module in $MODULES; do - (cd $module && mvn install -DskipTests -Dmaven.javadoc.skip=true) + (cd $module && mvn_install) done + +mvn_install -am -pl config-class-plugin -rf configgen +if $FULL; then + # Build all java modules required by C++ testing + mvn_install -am -pl filedistributionmanager,jrt,linguistics,messagebus -rf yolean +else + # Build minimal set of java modules required to run cmake + mvn_install -am -pl filedistributionmanager -rf yolean +fi diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java index fa5d46526f4..c8d2e868e42 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -70,6 +70,8 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui handleAttribute(e, t.searchNode); } else if (equals("summary", e)) { handleSummary(e, t.searchNode); + } else if (equals("initialize", e)) { + handleInitialize(e, t.searchNode); } } } @@ -244,4 +246,14 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } } + + private void handleInitialize(Element spec, Tuning.SearchNode sn) { + sn.initialize = new Tuning.SearchNode.Initialize(); + for (Element e : XML.getChildren(spec)) { + if (equals("threads", e)) { + sn.initialize.threads = asInt(e); + } + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 25a98353f74..1841f475e76 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -167,7 +167,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P } } - public static class Attribute implements ProtonConfig.Producer { + public static class Attribute implements ProtonConfig.Producer { public static class Io implements ProtonConfig.Producer { public IoType write = null; @@ -337,12 +337,24 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P } } + public static class Initialize implements ProtonConfig.Producer { + public Integer threads = null; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (threads != null) { + builder.initialize(new ProtonConfig.Initialize.Builder().threads(threads)); + } + } + } + public RequestThreads threads = null; public FlushStrategy strategy = null; public Resizing resizing = null; public Index index = null; public Attribute attribute = null; public Summary summary = null; + public Initialize initialize = null; @Override public void getConfig(ProtonConfig.Builder builder) { @@ -352,6 +364,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P if (index != null) index.getConfig(builder); if (attribute != null) attribute.getConfig(builder); if (summary != null) summary.getConfig(builder); + if (initialize != null) initialize.getConfig(builder); } } diff --git a/config-model/src/main/resources/schema/search.rnc b/config-model/src/main/resources/schema/search.rnc index 19a3860d93b..a96a4576f7b 100644 --- a/config-model/src/main/resources/schema/search.rnc +++ b/config-model/src/main/resources/schema/search.rnc @@ -221,6 +221,9 @@ Tuning = element tuning { }? }? }? + }? & + element initialize { + element threads { xsd:nonNegativeInteger }? }? }? } diff --git a/config-model/src/test/cfg/container/data/configserverinclude/hosted-vespa/hosted.xml b/config-model/src/test/cfg/container/data/configserverinclude/hosted-vespa/hosted.xml index dbe3bb659e0..fe4942d7c97 100644 --- a/config-model/src/test/cfg/container/data/configserverinclude/hosted-vespa/hosted.xml +++ b/config-model/src/test/cfg/container/data/configserverinclude/hosted-vespa/hosted.xml @@ -1,10 +1,3 @@ <!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <jdisc> - <config name="cloud.config.elk"> - <elasticsearch> - <item> - <host>foo</host> - </item> - </elasticsearch> - </config> </jdisc> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index caa4a34d47c..4a174970288 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -225,4 +225,14 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { assertThat(cfg, containsString("summary.log.chunk.compression.level 5")); } + @Test + public void requireThatWeCanParseInitializeTag() { + Tuning t = createTuning(parseXml("<initialize>", + "<threads>7</threads>", + "</initialize>")); + assertEquals(7, t.searchNode.initialize.threads.intValue()); + String cfg = getProtonCfg(t); + assertThat(cfg, containsString("initialize.threads 7")); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java deleted file mode 100644 index 7a8a554e650..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilderTest.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.test.MockRoot; -import com.yahoo.cloud.config.ElkConfig; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.configserver.TestOptions; -import org.junit.Test; - -import java.io.File; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author lulf - * @since 5.17 - */ -public class ConfigServerContainerModelBuilderTest { - @Test - public void testHostedVespaInclude() { - File testApp = new File("src/test/cfg/container/data/configserverinclude"); - FilesApplicationPackage app = FilesApplicationPackage.fromFile(testApp); - MockRoot root = new MockRoot(); - new ConfigServerContainerModelBuilder(new TestOptions()).build(new DeployState.Builder().applicationPackage(app).build(), null, root, XML.getChild(XML.getDocument(app.getServices()).getDocumentElement(), "jdisc")); - root.freezeModelTopology(); - ElkConfig config = root.getConfig(ElkConfig.class, "configserver/configserver"); - assertThat(config.elasticsearch().size(), is(1)); - assertThat(config.elasticsearch(0).host(), is("foo")); - } -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index a86dc68d9dc..87fb5e567a5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -31,6 +31,7 @@ import com.yahoo.vespa.config.UnknownConfigIdException; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; @@ -101,7 +102,7 @@ public class VespaModelTestCase { LogdConfig.Builder b = new LogdConfig.Builder(); b = (LogdConfig.Builder) model.getConfig(b, ""); LogdConfig c = new LogdConfig(b); - assertEquals(c.logserver().host(), LinuxInetAddress.getLocalHost().getCanonicalHostName()); + assertEquals(HostSystem.lookupCanonicalHostname(HostName.getLocalhost()), c.logserver().host()); SlobroksConfig.Builder sb = new SlobroksConfig.Builder(); sb = (com.yahoo.cloud.config.SlobroksConfig.Builder) model.getConfig(sb, ""); @@ -112,7 +113,7 @@ public class VespaModelTestCase { zb = (ZookeepersConfig.Builder) model.getConfig(zb, ""); ZookeepersConfig zc = new ZookeepersConfig(zb); assertEquals(zc.zookeeperserverlist().split(",").length, 2); - assertTrue(zc.zookeeperserverlist().startsWith(LinuxInetAddress.getLocalHost().getCanonicalHostName())); + assertTrue(zc.zookeeperserverlist().startsWith(HostSystem.lookupCanonicalHostname(HostName.getLocalhost()))); ApplicationIdConfig.Builder appIdBuilder = new ApplicationIdConfig.Builder(); appIdBuilder = (ApplicationIdConfig.Builder) model.getConfig(appIdBuilder, ""); diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 910c8da80ef..83e178bfa01 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -16,8 +16,6 @@ vespa_generate_config(configdefinitions configserver.def) install(FILES configserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions dispatch.def) install(FILES dispatch.def DESTINATION var/db/vespa/config_server/serverdb/classes) -vespa_generate_config(configdefinitions elk.def) -install(FILES elk.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions fleetcontroller.def) install(FILES fleetcontroller.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions ilscripts.def) diff --git a/configdefinitions/src/vespa/elk.def b/configdefinitions/src/vespa/elk.def deleted file mode 100644 index 0b144e84293..00000000000 --- a/configdefinitions/src/vespa/elk.def +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -# elasticsearch: for future use -elasticsearch[].host string - -# Port 0 (default) means use the Defaults web service port -elasticsearch[].port int default=0 - -# logstash forwarder -logstash.network.servers[].host string -logstash.network.servers[].port int default=5043 -logstash.network.timeout int default=15 -logstash.files[].paths[] string -logstash.files[].fields{} string - -# A relative path will be prepended by vespa home. -# An absolute path will be used as-is. -logstash.config_file string default="conf/logstash-forwarder/config.json" - -logstash.source_field string default="index_source" -logstash.spool_size int default=100 -# Note: verbose=true actually means "not noisy" in the forwarder -logstash.forwarder_command_line_params string default = "-verbose=true" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index f6b00440e30..4b94d4c21cc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -23,8 +23,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import com.yahoo.cloud.config.ElkConfig; - /** * Controls the lifetime of the {@link SuperModel} and the {@link SuperModelRequestHandler}. * @@ -32,6 +30,7 @@ import com.yahoo.cloud.config.ElkConfig; * @since 5.9 */ public class SuperModelController implements RequestHandler { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(SuperModelController.class.getName()); private volatile SuperModelRequestHandler handler; private final GenerationCounter generationCounter; @@ -39,14 +38,12 @@ public class SuperModelController implements RequestHandler { private final long masterGeneration; private final ConfigDefinitionRepo configDefinitionRepo; private final ConfigResponseFactory responseFactory; - private final ElkConfig elkConfig; private volatile boolean enabled = false; - public SuperModelController(GenerationCounter generationCounter, ConfigDefinitionRepo configDefinitionRepo, ConfigserverConfig configserverConfig, ElkConfig elkConfig) { + public SuperModelController(GenerationCounter generationCounter, ConfigDefinitionRepo configDefinitionRepo, ConfigserverConfig configserverConfig) { this.generationCounter = generationCounter; this.configDefinitionRepo = configDefinitionRepo; - this.elkConfig = elkConfig; this.masterGeneration = configserverConfig.masterGeneration(); this.responseFactory = ConfigResponseFactoryFactory.createFactory(configserverConfig); this.zone = new Zone(configserverConfig); @@ -85,7 +82,7 @@ public class SuperModelController implements RequestHandler { private SuperModelRequestHandler createNewHandler(Map<TenantName, Map<ApplicationId, Application>> newModels) { long generation = generationCounter.get() + masterGeneration; - SuperModel model = new SuperModel(newModels, elkConfig, zone); + SuperModel model = new SuperModel(newModels, zone); return new SuperModelRequestHandler(model, configDefinitionRepo, generation, responseFactory); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java index a36167517d8..78b89d7a3da 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java @@ -38,6 +38,7 @@ public class ApplicationConvergenceChecker extends AbstractComponent { private final static Set<String> serviceTypes = new HashSet<>(Arrays.asList( "container", + "container-clustercontroller", "qrserver", "docprocservice", "searchnode", diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java deleted file mode 100644 index 318a3f81d52..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.model; -import com.yahoo.cloud.config.ElkConfig.Builder; - -import com.yahoo.cloud.config.ElkConfig; -import com.yahoo.vespa.defaults.Defaults; - -/** - * Produces the ELK config for the SuperModel - * - * @author vegardh - * @since 5.38 - * - */ -public class ElkProducer implements ElkConfig.Producer { - - private final ElkConfig config; - - public ElkProducer(ElkConfig config) { - this.config = config; - } - - @Override - public void getConfig(Builder builder) { - for (ElkConfig.Elasticsearch es : config.elasticsearch()) { - int port = es.port() != 0 ? es.port() : Defaults.getDefaults().vespaWebServicePort(); - builder.elasticsearch(new ElkConfig.Elasticsearch.Builder().host(es.host()).port(port)); - } - ElkConfig.Logstash.Builder logstashBuilder = new ElkConfig.Logstash.Builder(); - logstashBuilder. - config_file(Defaults.getDefaults().underVespaHome(config.logstash().config_file())). - source_field(config.logstash().source_field()). - spool_size(config.logstash().spool_size()); - ElkConfig.Logstash.Network.Builder networkBuilder = new ElkConfig.Logstash.Network.Builder(). - timeout(config.logstash().network().timeout()); - for (ElkConfig.Logstash.Network.Servers srv : config.logstash().network().servers()) { - networkBuilder. - servers(new ElkConfig.Logstash.Network.Servers.Builder(). - host(srv.host()). - port(srv.port())); - } - logstashBuilder.network(networkBuilder); - for (ElkConfig.Logstash.Files files : config.logstash().files()) { - logstashBuilder.files(new ElkConfig.Logstash.Files.Builder(). - paths(files.paths()). - fields(files.fields())); - } - builder.logstash(logstashBuilder); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java index 2be7860b01f..1d34d4a0c00 100755 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.model; -import com.yahoo.cloud.config.ElkConfig; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.cloud.config.RoutingConfig; import com.yahoo.config.ConfigInstance; @@ -22,19 +21,16 @@ import java.util.Map; * * @author vegardh * @since 5.9 - * */ -public class SuperModel implements LbServicesConfig.Producer, ElkConfig.Producer, RoutingConfig.Producer { +public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Producer { private final Map<TenantName, Map<ApplicationId, Application>> models; private final LbServicesProducer lbProd; - private final ElkProducer elkProd; private final RoutingProducer zoneProd; - public SuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels, ElkConfig elkConfig, Zone zone) { + public SuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels, Zone zone) { this.models = newModels; this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(models), zone); - this.elkProd = new ElkProducer(elkConfig); this.zoneProd = new RoutingProducer(Collections.unmodifiableMap(models)); } @@ -44,10 +40,6 @@ public class SuperModel implements LbServicesConfig.Producer, ElkConfig.Producer LbServicesConfig.Builder builder = new LbServicesConfig.Builder(); getConfig(builder); return ConfigPayload.fromInstance(new LbServicesConfig(builder)); - } else if (configKey.equals(new ConfigKey<>(ElkConfig.class, configKey.getConfigId()))) { - ElkConfig.Builder builder = new ElkConfig.Builder(); - getConfig(builder); - return ConfigPayload.fromInstance(new ElkConfig(builder)); } else if (configKey.equals(new ConfigKey<>(RoutingConfig.class, configKey.getConfigId()))) { RoutingConfig.Builder builder = new RoutingConfig.Builder(); getConfig(builder); @@ -67,11 +59,6 @@ public class SuperModel implements LbServicesConfig.Producer, ElkConfig.Producer } @Override - public void getConfig(com.yahoo.cloud.config.ElkConfig.Builder builder) { - elkProd.getConfig(builder); - } - - @Override public void getConfig(RoutingConfig.Builder builder) { zoneProd.getConfig(builder); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index a69cf547db0..68bdf3150ab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -21,8 +21,6 @@ import java.io.File; import java.io.IOException; import java.util.Optional; -import com.yahoo.cloud.config.ElkConfig; - import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; @@ -43,8 +41,8 @@ public class SuperModelControllerTest { public void setup() throws IOException { counter = new SuperModelGenerationCounter(new MockCurator()); controller = new SuperModelController(counter, - new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder()), - new ElkConfig(new ElkConfig.Builder())); + new TestConfigDefinitionRepo(), + new ConfigserverConfig(new ConfigserverConfig.Builder())); } @Test @@ -94,11 +92,11 @@ public class SuperModelControllerTest { TenantName tenantA = TenantName.from("a"); long masterGen = 10; controller = new SuperModelController(counter, - new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen)), - new ElkConfig(new ElkConfig.Builder())); + new TestConfigDefinitionRepo(), + new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen))); long gen = counter.increment(); - controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3L, 1)); assertThat(controller.getHandler().getGeneration(), is(masterGen + gen)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 93a48094fac..e5c839abb01 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -2,20 +2,16 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.LbServicesConfig; -import com.yahoo.cloud.config.ElkConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.*; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications; import com.yahoo.vespa.config.protocol.CompressionType; -import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; -import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.model.SuperModel; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; @@ -32,9 +28,6 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; -import com.yahoo.cloud.config.ElkConfig.Logstash; - -import com.yahoo.vespa.config.server.model.ElkProducer; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -55,33 +48,7 @@ public class SuperModelRequestHandlerTest { ApplicationId app = ApplicationId.from(TenantName.from("a"), ApplicationName.from("foo"), InstanceName.defaultName()); models.get(app.tenant()).put(app, new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp)), new ServerCache(), 4l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app)); - handler = new SuperModelRequestHandler(new SuperModel(models, new ElkConfig(new ElkConfig.Builder()), Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); - } - - @Test - public void test_super_model_resolve_elk() { - ConfigResponse response = handler.resolveConfig(new GetConfigRequest() { - @Override - public ConfigKey<?> getConfigKey() { - return new ConfigKey<>(ElkConfig.class, "dontcare"); - } - - @Override - public DefContent getDefContent() { - return DefContent.fromClass(ElkConfig.class); - } - - @Override - public Optional<VespaVersion> getVespaVersion() { - return Optional.empty(); - } - - @Override - public boolean noCache() { - return false; - } - }); - assertThat(response.getGeneration(), is(2l)); + handler = new SuperModelRequestHandler(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); } @Test @@ -124,7 +91,7 @@ public class SuperModelRequestHandlerTest { models.get(TenantName.from("t2")).put(applicationId("minetooadvancedapp"), new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp3)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("minetooadvancedapp"))); - SuperModelRequestHandler han = new SuperModelRequestHandler(new SuperModel(models, new ElkConfig(new ElkConfig.Builder()), Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + SuperModelRequestHandler han = new SuperModelRequestHandler(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); LbServicesConfig lbc = new LbServicesConfig(lb); @@ -157,47 +124,7 @@ public class SuperModelRequestHandlerTest { org.junit.Assert.fail("No qrserver service in config"); } - @Test - public void testElkConfig() { - ElkConfig ec = new ElkConfig(new ElkConfig.Builder().elasticsearch(new ElkConfig.Elasticsearch.Builder().host("es1").port(99)). - logstash(new ElkConfig.Logstash.Builder(). - config_file("/cfgfile"). - source_field("srcfield"). - spool_size(345). - network(new Logstash.Network.Builder(). - servers(new Logstash.Network.Servers.Builder(). - host("ls1"). - port(999)). - servers(new Logstash.Network.Servers.Builder(). - host("ls2"). - port(998)). - timeout(78)). - files(new ElkConfig.Logstash.Files.Builder(). - paths("path1"). - paths("path2"). - fields("field1", "f1val"). - fields("field2", "f2val")))); - ElkProducer ep = new ElkProducer(ec); - ElkConfig.Builder newBuilder = new ElkConfig.Builder(); - ep.getConfig(newBuilder); - ElkConfig elkConfig = new ElkConfig(newBuilder); - assertThat(elkConfig.elasticsearch(0).host(), is("es1")); - assertThat(elkConfig.elasticsearch(0).port(), is(99)); - assertThat(elkConfig.logstash().network().servers(0).host(), is("ls1")); - assertThat(elkConfig.logstash().network().servers(0).port(), is(999)); - assertThat(elkConfig.logstash().network().servers(1).host(), is("ls2")); - assertThat(elkConfig.logstash().network().servers(1).port(), is(998)); - assertThat(elkConfig.logstash().network().timeout(), is(78)); - assertThat(elkConfig.logstash().config_file(), is("/cfgfile")); - assertThat(elkConfig.logstash().source_field(), is("srcfield")); - assertThat(elkConfig.logstash().spool_size(), is(345)); - assertThat(elkConfig.logstash().files().size(), is(1)); - assertThat(elkConfig.logstash().files(0).paths(0), is("path1")); - assertThat(elkConfig.logstash().files(0).paths(1), is("path2")); - assertThat(elkConfig.logstash().files(0).fields("field1"), is("f1val")); - assertThat(elkConfig.logstash().files(0).fields("field2"), is("f2val")); - } - } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java index 3e5431deccd..30143f565e5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.cloud.config.ElkConfig; import com.yahoo.config.provision.TenantName; import com.yahoo.jrt.Request; import com.yahoo.jrt.Spec; @@ -65,7 +64,7 @@ public class TestWithRpc { protected void createAndStartRpcServer(boolean hostedVespa) { rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder().rpcport(port).numthreads(1).maxgetconfigclients(1).hostedVespa(hostedVespa)), new SuperModelController(generationCounter, - new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder()), new ElkConfig(new ElkConfig.Builder())), + new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder())), Metrics.createTestMetrics(), new HostRegistries()); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); t = new Thread(rpcServer); diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh new file mode 100755 index 00000000000..f76f8d70455 --- /dev/null +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -0,0 +1,159 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#set -x + +if [ -z "${VESPA_HOME}" ]; then + echo "Missing VESPA_HOME variable" + exit 1 +fi +if [ -z "${VESPA_SERVICE_NAME}" ]; then + echo "Missing VESPA_SERVICE_NAME variable" + exit 1 +fi +if [ -z "${VESPA_CONFIG_ID}" ]; then + echo "Missing VESPA_CONFIG_ID variable" + exit 1 +fi +cd ${VESPA_HOME} || { echo "Cannot cd to ${VESPA_HOME}" 1>&2; exit 1; } + +. libexec/vespa/common-env.sh + +DISCRIMINATOR=`echo ${VESPA_CONFIG_ID} | md5sum | cut -d' ' -f1` +CONTAINER_HOME="${VESPA_HOME}var/jdisc_container/${DISCRIMINATOR}/" + +ZOOKEEPER_LOG_FILE="${VESPA_HOME}logs/vespa/zookeeper.${VESPA_SERVICE_NAME}.log" +rm -f $ZOOKEEPER_LOG_FILE*lck + +# common setup +export VESPA_LOG_TARGET=file:${VESPA_HOME}logs/vespa/vespa.log +export VESPA_LOG_CONTROL_DIR=${VESPA_HOME}var/db/vespa/logcontrol +export LD_LIBRARY_PATH=${VESPA_HOME}lib64 + +cfpfile=${CONTAINER_HOME}/jdisc.properties +bundlecachedir=${CONTAINER_HOME}/bundlecache + +# class path +CP="${VESPA_HOME}lib/jars/jdisc_core-jar-with-dependencies.jar" + +mkdir -p $bundlecachedir || exit 1 +printenv > $cfpfile || exit 1 + +# ??? TODO ??? XXX ??? +# LANG=en_US.utf8 +# LC_ALL=C + + +getconfig() { + qrstartcfg="" + case "${VESPA_CONFIG_ID}" in + dir:*) + config_dir=${VESPA_CONFIG_ID#dir:} + qrstartcfg="`cat ${config_dir}/qr-start.cfg`" + ;; + *) + qrstartcfg="`getvespaconfig -w 10 -n search.config.qr-start -i ${VESPA_CONFIG_ID}`" + ;; + esac + cmds=`echo "$qrstartcfg" | perl -ne 's/^(\w+)\.(\w+) (.*)/$1_$2=$3/ && print'` + eval "$cmds" +} + +configure_memory() { + consider_fallback jvm_heapsize 1536 + consider_fallback jvm_stacksize 512 + consider_fallback jvm_baseMaxDirectMemorySize 75 + consider_fallback jvm_directMemorySizeCache 0 + + if (( jvm_heapSizeAsPercentageOfPhysicalMemory > 0 && jvm_heapSizeAsPercentageOfPhysicalMemory < 100 )); then + available=`free -m | grep Mem | tr -s ' ' | cut -f2 -d' '` + jvm_heapsize=$[available * jvm_heapSizeAsPercentageOfPhysicalMemory / 100] + if (( jvm_heapsize < 1024 )); then + jvm_heapsize=1024 + fi + fi + maxDirectMemorySize=$(( ${jvm_baseMaxDirectMemorySize} + ${jvm_heapsize}/8 + ${jvm_directMemorySizeCache} )) + + memory_options="-Xms${jvm_heapsize}m -Xmx${jvm_heapsize}m" + memory_options="${memory_options} -XX:ThreadStackSize=${jvm_stacksize}" + memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m" + + if [ "${VESPA_USE_HUGEPAGES}" ]; then + memory_options="${memory_options} -XX:+UseLargePages" + fi +} + +configure_gcopts() { + consider_fallback jvm_gcopts "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1" + if [ "$jvm_verbosegc" = "true" ]; then + jvm_gcopts="${jvm_gcopts} -verbose:gc" + fi +} + +configure_env_vars() { + if [ "$qrs_env" ]; then + for setting in ${qrs_env} ; do + case $setting in + *"="*) + eval "$setting"; + export ${setting%%=*} + ;; + *) + echo "warning ignoring invalid qrs_env setting '$setting' from '$qrs_env'" + ;; + esac + done + fi +} + +configure_classpath () { + if [ "${jdisc_classpath_extra}" ]; then + CP="${CP}:${jdisc_classpath_extra}" + fi +} + +configure_preload () { + export JAVAVM_LD_PRELOAD= + unset LD_PRELOAD + if [ "$PRELOAD" ]; then + export JAVAVM_LD_PRELOAD="$PRELOAD" + export LD_PRELOAD="$PRELOAD" + fi +} + +getconfig +configure_memory +configure_gcopts +configure_env_vars +configure_classpath +# note: should be last thing here: +configure_preload + +exec java \ + -Dconfig.id="${VESPA_CONFIG_ID}" \ + ${memory_options} \ + ${jvm_gcopts} \ + -XX:MaxJavaStackTraceDepth=-1 \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:HeapDumpPath="${VESPA_HOME}var/crash" \ + -XX:OnOutOfMemoryError='kill -9 %p' \ + -Djava.library.path="${VESPA_HOME}lib64" \ + -Djava.awt.headless=true \ + -Djavax.net.ssl.keyStoreType=JKS \ + -Dsun.rmi.dgc.client.gcInterval=3600000 \ + -Dsun.net.client.defaultConnectTimeout=5000 -Dsun.net.client.defaultReadTimeout=60000 \ + -Djdisc.config.file="$cfpfile" \ + -Djdisc.export.packages=${jdisc_export_packages} \ + -Djdisc.cache.path="$bundlecachedir" \ + -Djdisc.debug.resources=false \ + -Djdisc.bundle.path="${VESPA_HOME}lib/jars" \ + -Djdisc.logger.enabled=true \ + -Djdisc.logger.level=ALL \ + -Djdisc.logger.tag="${VESPA_CONFIG_ID}" \ + -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger \ + -Dvespa.log.control.dir="${VESPA_LOG_CONTROL_DIR}" \ + -Dzookeeperlogfile="${ZOOKEEPER_LOG_FILE}" \ + -Dfile.encoding=UTF-8 \ + -cp "$CP" \ + "$@" \ + com.yahoo.jdisc.core.StandaloneMain file:${VESPA_HOME}lib/jars/container-disc-jar-with-dependencies.jar diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index f6e8ee1f27a..653589a8e91 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -25,8 +25,8 @@ import edu.umd.cs.findbugs.annotations.NonNull; * Ensure the fields specified in {@link Presentation#getSummaryFields()} are * available after filling phase. * - * @author <a href="mailto:stiankri@yahoo-inc.com">Stian Kristoffersen</a> - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author stiankri + * @author Steinar Knutsen */ @Beta @After(MinimalQueryInserter.EXTERNAL_YQL) diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index a7cc06c95f7..bcdc6ba060f 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -84,9 +84,9 @@ import edu.umd.cs.findbugs.annotations.NonNull; * VespaSerializer. * </p> * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @author <a href="mailto:stiankri@yahoo-inc.com">Stian Kristoffersen</a> - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Steinar Knutsen + * @author stiankri + * @author Simon Thoresen */ @Beta public class YqlParser implements Parser { diff --git a/dist/vespa.spec b/dist/vespa.spec index 4a37716759f..04de38731a3 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -120,7 +120,7 @@ mkdir -p %{buildroot}/%{_prefix}/var/spool/master/inbox/ mkdir -p %{buildroot}/%{_prefix}/var/vespa/bundlecache/ mkdir -p %{buildroot}/%{_prefix}/var/vespa/cache/config/ mkdir -p %{buildroot}/%{_prefix}/var/vespa/cmdlines/ -mkdir -p %{buildroot}/%{_prefix}/var/zookeeper/ +mkdir -p %{buildroot}/%{_prefix}/var/zookeeper/version-2/ ln -s %{_prefix}/lib/jars/config-model-fat.jar %{buildroot}/%{_prefix}/conf/configserver-app/components/config-model-fat.jar ln -s %{_prefix}/lib/jars/configserver-jar-with-dependencies.jar %{buildroot}/%{_prefix}/conf/configserver-app/components/configserver.jar @@ -185,7 +185,7 @@ exit 0 %dir %attr( 755, vespa, vespa) %{_prefix}/var/vespa/bundlecache/ %dir %attr( 755, vespa, vespa) %{_prefix}/var/vespa/cache/config/ %dir %attr( 755, vespa, vespa) %{_prefix}/var/vespa/cmdlines/ -%dir %attr( 755, vespa, vespa) %{_prefix}/var/zookeeper/ +%dir %attr( 755, vespa, vespa) %{_prefix}/var/zookeeper/version-2/ %{_prefix}/libexec/vespa/vespa-config.pl %{_prefix}/libexec/vespa/common-env.sh diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build new file mode 100644 index 00000000000..0f7f6935042 --- /dev/null +++ b/docker/Dockerfile.build @@ -0,0 +1,34 @@ +FROM centos:7 + +# Needed to build vespa +RUN yum -y install epel-release +RUN yum -y install centos-release-scl +RUN yum -y install devtoolset-4-gcc-c++ +RUN yum -y install devtoolset-4-libatomic-devel +RUN yum -y install Judy-devel +RUN yum -y install cmake3 +RUN yum -y install ccache +RUN yum -y install lz4-devel +RUN yum -y install zlib-devel +RUN yum -y install maven +RUN yum -y install libicu-devel +RUN yum -y install llvm-devel +RUN yum -y install llvm-static +RUN yum -y install java-1.8.0-openjdk-devel +RUN yum -y install openssl-devel +RUN yum -y install rpm-build +RUN yum -y install make + +# Install vespa dependencies +RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo +RUN yum -y install vespa-boost-devel +RUN yum -y install vespa-libtorrent-devel +RUN yum -y install vespa-zookeeper-c-client-devel +RUN yum -y install vespa-cppunit-devel + +# Install utilities +RUN yum -y install sudo + +# Enable devtoolset-4 by default +RUN echo "source /opt/rh/devtoolset-4/enable" > /etc/profile.d/devtoolset-4.sh + diff --git a/docker/Dockerfile.run b/docker/Dockerfile.run new file mode 100644 index 00000000000..d82297ce676 --- /dev/null +++ b/docker/Dockerfile.run @@ -0,0 +1,27 @@ +FROM centos:7 + +# Needed to build vespa +RUN yum -y install epel-release +RUN yum -y install centos-release-scl +RUN yum -y install Judy +RUN yum -y install lz4 +RUN yum -y install zlib +RUN yum -y install libicu +RUN yum -y install llvm +RUN yum -y install java-1.8.0-openjdk +RUN yum -y install openssl +RUN yum -y install perl +RUN yum -y install perl-Env +RUN yum -y install perl-JSON +RUN yum -y install libatomic + +# Install vespa dependencies +RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo +RUN yum -y install vespa-boost +RUN yum -y install vespa-libtorrent +RUN yum -y install vespa-zookeeper-c-client +RUN yum -y install vespa-cppunit # Should not be needed ? + +# Utilities +RUN yum -y install net-tools less + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..59424ff2d98 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,33 @@ + +# Building and running Vespa on Docker (OS X and Linux) + +## Installing docker +[Docker installation](https://docs.docker.com/engine/installation/) + +*On OS X, the native Docker engine (Beta) has NOT been tested. Please use the [Docker Toolbox](https://www.docker.com/products/docker-toolbox).* + +*On Linux, the default storage device is devicemapper with loopback device and max 10GB container size. This size is too small for a full build. Please see [here](http://www.projectatomic.io/blog/2016/03/daemon_option_basedevicesize/) and [here](http://www.projectatomic.io/blog/2015/06/notes-on-fedora-centos-and-docker-storage-drivers/) to overcome this limitation.* + +## Building the Vespa RPM +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./build-vespa.sh <Vespa version number>``` to build Vespa from this source code. + +The produced rpms will be available in this folder after compiliation. + +## Running Vespa +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./run-vespa.sh <Vespa version number>``` to start Vespa. + +This will create a Docker image which has the rpms from the build step (or downloaded rpms to this folder) installed. Vespa will be started inside the container. + +*On OS X, the container runs inside the Docker VM. Execute ```docker-machine ssh vespa-docker-machine``` to enter the VM. The services can also be reached directly from the host on the IP given by ```docker-machine ip vespa-docker-machine```* + +## Building Vespa inside a Docker container +*On OS X, execute ```source osx-setup-docker-machine.sh``` to setup the Docker VM in which to run Docker.* + +Execute ```./enter-build-container.sh``` to enter the Vespa build environment inside a Docker container. + +The container is entered at the root of the Vespa source repository. Follow the build sections in [README.md](https://github.com/yahoo/vespa/blob/master/README.md) to build and test. + diff --git a/docker/build-vespa-internal.sh b/docker/build-vespa-internal.sh new file mode 100755 index 00000000000..f79e936c800 --- /dev/null +++ b/docker/build-vespa-internal.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +if [ $# -ne 3 ]; then + echo "Usage: $0 <vespa version> <caller uid> <caller gid>" + echo "This script should not be called manually." + exit 1 +fi +VESPA_VERSION=$1 +CALLER_UID=$2 +CALLER_GID=$3 + +cd /vespa +./dist.sh ${VESPA_VERSION} +rpmbuild -bb ~/rpmbuild/SPECS/vespa-${VESPA_VERSION}.spec +chown ${CALLER_UID}:${CALLER_GID} ~/rpmbuild/RPMS/x86_64/*.rpm +mv ~/rpmbuild/RPMS/x86_64/*.rpm /vespa/docker + diff --git a/docker/build-vespa.sh b/docker/build-vespa.sh new file mode 100755 index 00000000000..6d3b1699bc5 --- /dev/null +++ b/docker/build-vespa.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 +DOCKER_IMAGE="vespabuild" + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.build . +docker run --rm -v $(pwd)/..:/vespa --entrypoint /vespa/docker/build-vespa-internal.sh "$DOCKER_IMAGE" "$VESPA_VERSION" "$(id -u)" "$(id -g)" + diff --git a/docker/enter-build-container-internal.sh b/docker/enter-build-container-internal.sh new file mode 100755 index 00000000000..7da96fde376 --- /dev/null +++ b/docker/enter-build-container-internal.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +if [ $# -ne 0 ]; then + echo "Usage: $0" + echo "This script should not be called manually." + exit 1 +fi + +USERNAME=builder +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +CALLER_UID=$(stat -c "%u" $DIR) +CALLER_GID=$(stat -c "%g" $DIR) + +groupadd -f -g $CALLER_GID $USERNAME +useradd -u $CALLER_UID -g $CALLER_GID $USERNAME +echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +su -c "mkdir -p $DIR/../.ccache" $USERNAME +su -c "ln -sf $DIR/../.ccache /home/$USERNAME/.ccache" $USERNAME + +su -c "mkdir -p $DIR/../.m2" $USERNAME +su -c "ln -sf $DIR/../.m2 /home/$USERNAME/.m2" $USERNAME + +cd $DIR/.. +su $USERNAME + diff --git a/docker/enter-build-container.sh b/docker/enter-build-container.sh new file mode 100755 index 00000000000..ed7a2b4a130 --- /dev/null +++ b/docker/enter-build-container.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +if [ $# -ne 0 ]; then + echo "Usage: $0" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +DOCKER_IMAGE="vespabuild" + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.build . +docker run -ti --rm -v $(pwd)/..:/vespa --entrypoint /vespa/docker/enter-build-container-internal.sh "$DOCKER_IMAGE" + diff --git a/docker/osx-setup-docker-machine.sh b/docker/osx-setup-docker-machine.sh new file mode 100755 index 00000000000..f6c9e870cf8 --- /dev/null +++ b/docker/osx-setup-docker-machine.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +DOCKER_VM_NAME=vespa-docker-machine +DOCKER_VM_DISK_SIZE_IN_MB=40000 +DOCKER_VM_MEMORY_SIZE_IN_MB=4000 +DOCKER_VM_CPU_COUNT=4 + +DOCKER_VM_WAS_STARTED=false + +if ! docker-machine status "$DOCKER_VM_NAME" &> /dev/null; then + # Machine does not exist and we have to create and start + docker-machine create -d virtualbox \ + --virtualbox-disk-size "$DOCKER_VM_DISK_SIZE_IN_MB" \ + --virtualbox-memory "$DOCKER_VM_MEMORY_SIZE_IN_MB" \ + --virtualbox-cpu-count "$DOCKER_VM_CPU_COUNT" \ + "$DOCKER_VM_NAME" + + eval $(docker-machine env "$DOCKER_VM_NAME") + DOCKER_VM_WAS_STARTED=true +fi + + +VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME") +if [ "$VESPA_VM_STATUS" = "Stopped" ]; then + docker-machine start "$DOCKER_VM_NAME" + DOCKER_VM_WAS_STARTED=true + VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME") +fi + +if [ "$VESPA_VM_STATUS" != "Running" ]; then + echo "Unable to get Docker machine $DOCKER_VM_NAME up and running." + echo "You can try to manually remove the machine: docker-machine rm -y $DOCKER_VM_NAME " + echo " and then rerun this script." + echo "Exiting." + exit 1 +fi + +if $DOCKER_VM_WAS_STARTED; then + # Hostname should match the public IP + docker-machine ssh "$DOCKER_VM_NAME" "sudo sed -i \"s/127.0.0.1 $DOCKER_VM_NAME/127.0.0.1/\" /etc/hosts" + docker-machine ssh "$DOCKER_VM_NAME" "sudo sed -i \"/$DOCKER_VM_NAME/d\" /etc/hosts" + docker-machine ssh "$DOCKER_VM_NAME" "sudo echo $(docker-machine ip $DOCKER_VM_NAME) $DOCKER_VM_NAME | sudo tee -a /etc/hosts" > /dev/null +fi + +eval $(docker-machine env "$DOCKER_VM_NAME") + diff --git a/docker/run-vespa-internal.sh b/docker/run-vespa-internal.sh new file mode 100755 index 00000000000..f019f1a3740 --- /dev/null +++ b/docker/run-vespa-internal.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + echo "This script should not be called manually." + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 + +rpm -i "vespa*-${VESPA_VERSION}-*.rpm" + +# Workaround until we figure out why rpm does not set the ownership. +chown -R vespa:vespa /opt/vespa + +/opt/vespa/bin/vespa-start-configserver +/opt/vespa/bin/vespa-start-services + +# Print log forever +while true; do + /opt/vespa/bin/logfmt -f /opt/vespa/logs/vespa/vespa.log + sleep 10 +done diff --git a/docker/run-vespa.sh b/docker/run-vespa.sh new file mode 100755 index 00000000000..e11324a7b93 --- /dev/null +++ b/docker/run-vespa.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <vespa version>" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +VESPA_VERSION=$1 +DOCKER_IMAGE=vesparun + +docker build -t "$DOCKER_IMAGE" -f Dockerfile.run . +docker run -d -v $(pwd)/..:/vespa --net=host --privileged --entrypoint /vespa/docker/run-vespa-internal.sh "$DOCKER_IMAGE" "$VESPA_VERSION" + diff --git a/install_java.cmake b/install_java.cmake index cc618630cd0..27ba82472f5 100644 --- a/install_java.cmake +++ b/install_java.cmake @@ -18,6 +18,11 @@ install_java_artifact_dependencies(vespa_jersey2) install_java_artifact(vespajlib) install_fat_java_artifact(application-preprocessor) +install_fat_java_artifact(clustercontroller-apps) +install_fat_java_artifact(clustercontroller-apputil) +install_fat_java_artifact(clustercontroller-utils) +install_fat_java_artifact(clustercontroller-core) + install_fat_java_artifact(component) install_fat_java_artifact(config-bundle) install_fat_java_artifact(config-model-api) @@ -50,6 +55,8 @@ vespa_install_script(config-model/src/main/perl/deploy-application bin) vespa_install_script(config-model/src/main/perl/expand-config.pl bin) vespa_install_script(config-model/src/main/perl/vespa-replicate-log-stream bin) vespa_install_script(config-model/src/main/sh/validate-application bin) +vespa_install_script(container-disc/src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) + vespa_install_script(logserver/bin/logserver-start.sh logserver-start bin) install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) diff --git a/jdisc_akamai/OWNERS b/jdisc_akamai/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_akamai/OWNERS +++ b/jdisc_akamai/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_container_maven_archetype_application/OWNERS b/jdisc_container_maven_archetype_application/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_container_maven_archetype_application/OWNERS +++ b/jdisc_container_maven_archetype_application/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_core/OWNERS b/jdisc_core/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_core/OWNERS +++ b/jdisc_core/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java index 84e91a52e3a..19b6c220f07 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java @@ -36,16 +36,17 @@ public class StandaloneMain { void run(String bundleLocation) { try { - log.info("Initializing application without privileges."); - loader.init(bundleLocation, false); - loader.start(); - setupSigTermHandler(); - waitForShutdown(); - // Event.stopping(APPNAME, "shutdown"); - loader.stop(); - // Event.stopped(APPNAME, 0, 0); - loader.destroy(); - // System.exit(0); + System.out.println("debug\tInitializing application without privileges."); + loader.init(bundleLocation, false); + loader.start(); + setupSigTermHandler(); + waitForShutdown(); + System.out.println("debug\tTrying to shutdown in a controlled manner."); + loader.stop(); + System.out.println("debug\tTrying to clean up in a controlled manner."); + loader.destroy(); + System.out.println("debug\tStopped ok."); + System.exit(0); } catch (Exception e) { log.log(Level.WARNING, "Unexpected: ", e); System.exit(6); diff --git a/jdisc_core_test/OWNERS b/jdisc_core_test/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_core_test/OWNERS +++ b/jdisc_core_test/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_http_service/OWNERS b/jdisc_http_service/OWNERS index 5255d2560bb..6c536000692 100644 --- a/jdisc_http_service/OWNERS +++ b/jdisc_http_service/OWNERS @@ -1,2 +1,3 @@ bakksjo gjoranv +bjorncs diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index e9aba0cb6c9..c16ac589332 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -101,6 +101,13 @@ class HttpRequestDispatch { HttpRequestDispatch parent = this; //used to avoid binding uninitialized variables completeRequestCallback = (result, error) -> { + boolean alreadyCalled = completeRequestCalled.getAndSet(true); + if (alreadyCalled) { + AssertionError e = new AssertionError("completeRequest called more than once"); + log.log(Level.WARNING, "Assertion failed.", e); + throw e; + } + boolean reportedError = false; if (error != null) { @@ -113,14 +120,6 @@ class HttpRequestDispatch { parent.metricReporter.successfulResponse(); } - - boolean alreadyCalled = completeRequestCalled.getAndSet(true); - if (alreadyCalled) { - AssertionError e = new AssertionError("completeRequest called more than once"); - log.log(Level.WARNING, "Assertion failed.", e); - throw e; - } - try { parent.async.complete(); log.finest(() -> "Request completed successfully: " + parent.servletRequest.getRequestURI()); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java index 271805765c2..7a15107a3eb 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletOutputStreamWriter.java @@ -41,6 +41,10 @@ public class ServletOutputStreamWriter { private static final Logger log = Logger.getLogger(ServletOutputStreamWriter.class.getName()); + // TODO: This reference is not guaranteed to be unique; ByteBuffer.allocate(0) MAY in principle return a singleton! + // If so, application code could fake a close by writing such a byte buffer. + // The problem can be solved by filtering out zero-length byte buffers from application code. + // Other ways to express this are also possible, e.g. with a 'closed' state checked when queue goes empty. private static final ByteBuffer CLOSE_STREAM_BUFFER = ByteBuffer.allocate(0); private final Object monitor = new Object(); @@ -74,6 +78,7 @@ public class ServletOutputStreamWriter { public void setSendingError() { synchronized (monitor) { + // TODO: This assert seems fishy. Investigate. assertStateIs(state, State.NOT_STARTED); state = State.FINISHED_OR_ERROR; } @@ -109,6 +114,7 @@ public class ServletOutputStreamWriter { } if (thisThreadShouldWrite) { + // TODO: Consider refactoring to avoid multiple monitor entry-exit. writeBuffersInQueueToOutputStream(); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index 5bea01bd104..a763a03d39d 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -133,9 +133,8 @@ class ServletRequestReader implements ReadListener { numberOfOutstandingUserCalls += 2; } try { - requestContentChannel.write(buf, writeCompletionHandler); - int bytesReceived = buf.remaining(); + requestContentChannel.write(buf, writeCompletionHandler); metricReporter.successfulRead(bytesReceived); } catch (final Throwable t) { finishedFuture.completeExceptionally(t); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java index b0781c402d5..126e4fee9e6 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletResponseController.java @@ -115,6 +115,8 @@ public class ServletResponseController { } try { + // TODO: sendError() is a synchronous call. Refactor. (Also, we should control the response content - + // this method generates a response body based on Jetty's own response templates ("Powered by Jetty"). servletResponse.sendError( statusCode, reasonPhrase); @@ -166,10 +168,14 @@ public class ServletResponseController { private static void setStatus_holdingLock(Response jdiscResponse, HttpServletResponse servletResponse) { if (jdiscResponse instanceof HttpResponse) { + // TODO: Figure out what this does to the response (with Jetty), and move to non-deprecated APIs. + // Deprecate our own code as necessary. servletResponse.setStatus(jdiscResponse.getStatus(), ((HttpResponse) jdiscResponse).getMessage()); } else { Optional<String> errorMessage = getErrorMessage(jdiscResponse); if (errorMessage.isPresent()) { + // TODO: Figure out what this does to the response (with Jetty), and move to non-deprecated APIs. + // Deprecate our own code as necessary. servletResponse.setStatus(jdiscResponse.getStatus(), errorMessage.get()); } else { servletResponse.setStatus(jdiscResponse.getStatus()); diff --git a/jdisc_jetty/OWNERS b/jdisc_jetty/OWNERS index 5255d2560bb..6c536000692 100644 --- a/jdisc_jetty/OWNERS +++ b/jdisc_jetty/OWNERS @@ -1,2 +1,3 @@ bakksjo gjoranv +bjorncs diff --git a/jdisc_jmx_metrics/OWNERS b/jdisc_jmx_metrics/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_jmx_metrics/OWNERS +++ b/jdisc_jmx_metrics/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_maven_archetype_component/OWNERS b/jdisc_maven_archetype_component/OWNERS index 3b2ba1ede81..78b92e411b4 100644 --- a/jdisc_maven_archetype_component/OWNERS +++ b/jdisc_maven_archetype_component/OWNERS @@ -1 +1,2 @@ gjoranv +bjorncs diff --git a/jdisc_messagebus_service/OWNERS b/jdisc_messagebus_service/OWNERS index 90fdb511ae3..6c536000692 100644 --- a/jdisc_messagebus_service/OWNERS +++ b/jdisc_messagebus_service/OWNERS @@ -1 +1,3 @@ bakksjo +gjoranv +bjorncs diff --git a/jdisc_status/OWNERS b/jdisc_status/OWNERS index d24d7c7860d..f02be9b1f86 100644 --- a/jdisc_status/OWNERS +++ b/jdisc_status/OWNERS @@ -1,2 +1,3 @@ gjoranv bakksjo +bjorncs diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index f2b31b3afb9..94e9a405ee4 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -1,13 +1,13 @@ <?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0"> <jdisc version="1.0" jetty="true"> - <rest-api path="test" jersey2="true"> - <components bundle="node-admin"> - <package>com.yahoo.vespa.hosted.node.admin.testapi</package> - </components> - </rest-api> - <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.NodeAdminScheduler" bundle="node-admin"/> + <!-- Please update container test when changing this file --> + <handler id="com.yahoo.vespa.hosted.node.admin.restapi" bundle="node-admin"> + <binding>http://*/rest/*</binding> + </handler> + <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl" bundle="node-admin"/> + <component id="docker" class="com.yahoo.vespa.hosted.node.admin.docker.DockerImpl" bundle="node-admin"/> + <config name='nodeadmin.docker.docker'> <caCertPath>/host/docker/certs/ca_cert.pem</caCertPath> <clientCertPath>/host/docker/certs/client_cert.pem</clientCertPath> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java index 6d4873d92bf..147f7cdab9f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java @@ -1,155 +1,23 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin; -import com.yahoo.collections.Pair; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.hosted.node.admin.docker.Container; -import com.yahoo.vespa.hosted.node.admin.docker.Docker; -import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** - * The "most important" class in this module, where the main business logic resides or is driven from. - * - * @author stiankri + * API for NodeAdmin seen from outside. + * @author dybis */ -public class NodeAdmin { - private static final Logger logger = Logger.getLogger(NodeAdmin.class.getName()); +public interface NodeAdmin { - private static final long MIN_AGE_IMAGE_GC_MILLIS = Duration.ofMinutes(15).toMillis(); + void setState(final List<ContainerNodeSpec> containersToRun); - private final Docker docker; - private final Function<HostName, NodeAgent> nodeAgentFactory; + boolean freezeAndCheckIfAllFrozen(); - private final Map<HostName, NodeAgent> nodeAgents = new HashMap<>(); + void unfreeze(); - private Map<DockerImage, Long> firstTimeEligibleForGC = Collections.emptyMap(); + Set<HostName> getListOfHosts(); - /** - * @param docker interface to docker daemon and docker-related tasks - * @param nodeAgentFactory factory for {@link NodeAgent} objects - */ - public NodeAdmin(final Docker docker, final Function<HostName, NodeAgent> nodeAgentFactory) { - this.docker = docker; - this.nodeAgentFactory = nodeAgentFactory; - } - - public void maintainWantedState(final List<ContainerNodeSpec> containersToRun) { - final List<Container> existingContainers = docker.getAllManagedContainers(); - - synchronizeLocalContainerState(containersToRun, existingContainers); - - garbageCollectDockerImages(containersToRun); - } - - private void garbageCollectDockerImages(final List<ContainerNodeSpec> containersToRun) { - final Set<DockerImage> deletableDockerImages = getDeletableDockerImages( - docker.getUnusedDockerImages(), containersToRun); - final long currentTime = System.currentTimeMillis(); - // TODO: This logic should be unit tested. - firstTimeEligibleForGC = deletableDockerImages.stream() - .collect(Collectors.toMap( - dockerImage -> dockerImage, - dockerImage -> Optional.ofNullable(firstTimeEligibleForGC.get(dockerImage)).orElse(currentTime))); - // Delete images that have been eligible for some time. - firstTimeEligibleForGC.forEach((dockerImage, timestamp) -> { - if (currentTime - timestamp > MIN_AGE_IMAGE_GC_MILLIS) { - docker.deleteImage(dockerImage); - } - }); - } - - // Turns an Optional<T> into a Stream<T> of length zero or one depending upon whether a value is present. - // This is a workaround for Java 8 not having Stream.flatMap(Optional). - private static <T> Stream<T> streamOf(Optional<T> opt) { - return opt.map(Stream::of) - .orElseGet(Stream::empty); - } - - static Set<DockerImage> getDeletableDockerImages( - final Set<DockerImage> currentlyUnusedDockerImages, - final List<ContainerNodeSpec> pendingContainers) { - final Set<DockerImage> imagesNeededNowOrInTheFuture = pendingContainers.stream() - .flatMap(nodeSpec -> streamOf(nodeSpec.wantedDockerImage)) - .collect(Collectors.toSet()); - return diff(currentlyUnusedDockerImages, imagesNeededNowOrInTheFuture); - } - - // Set-difference. Returns minuend minus subtrahend. - private static <T> Set<T> diff(final Set<T> minuend, final Set<T> subtrahend) { - final HashSet<T> result = new HashSet<>(minuend); - result.removeAll(subtrahend); - return result; - } - - // Returns a full outer join of two data sources (of types T and U) on some extractable attribute (of type V). - // Full outer join means that all elements of both data sources are included in the result, - // even when there is no corresponding element (having the same attribute) in the other data set, - // in which case the value from the other source will be empty. - static <T, U, V> Stream<Pair<Optional<T>, Optional<U>>> fullOuterJoin( - final Stream<T> tStream, final Function<T, V> tAttributeExtractor, - final Stream<U> uStream, final Function<U, V> uAttributeExtractor) { - final Map<V, T> tMap = tStream.collect(Collectors.toMap(tAttributeExtractor, t -> t)); - final Map<V, U> uMap = uStream.collect(Collectors.toMap(uAttributeExtractor, u -> u)); - return Stream.concat(tMap.keySet().stream(), uMap.keySet().stream()) - .distinct() - .map(key -> new Pair<>(Optional.ofNullable(tMap.get(key)), Optional.ofNullable(uMap.get(key)))); - } - - void synchronizeLocalContainerState( - final List<ContainerNodeSpec> containersToRun, - final List<Container> existingContainers) { - final Stream<Pair<Optional<ContainerNodeSpec>, Optional<Container>>> nodeSpecContainerPairs = fullOuterJoin( - containersToRun.stream(), nodeSpec -> nodeSpec.hostname, - existingContainers.stream(), container -> container.hostname); - - final Set<HostName> nodeHostNames = containersToRun.stream() - .map(spec -> spec.hostname) - .collect(Collectors.toSet()); - final Set<HostName> obsoleteAgentHostNames = diff(nodeAgents.keySet(), nodeHostNames); - obsoleteAgentHostNames.forEach(hostName -> nodeAgents.remove(hostName).stop()); - - nodeSpecContainerPairs.forEach(nodeSpecContainerPair -> { - final Optional<ContainerNodeSpec> nodeSpec = nodeSpecContainerPair.getFirst(); - final Optional<Container> existingContainer = nodeSpecContainerPair.getSecond(); - - if (!nodeSpec.isPresent()) { - assert existingContainer.isPresent(); - logger.warning("Container " + existingContainer.get() + " exists, but is not in node repository runlist"); - return; - } - - try { - updateAgent(nodeSpec.get()); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed to bring container to desired state", e); - } - }); - } - - private void updateAgent(final ContainerNodeSpec nodeSpec) throws IOException { - final NodeAgent agent; - if (nodeAgents.containsKey(nodeSpec.hostname)) { - agent = nodeAgents.get(nodeSpec.hostname); - } else { - agent = nodeAgentFactory.apply(nodeSpec.hostname); - nodeAgents.put(nodeSpec.hostname, agent); - agent.start(); - } - agent.update(); - } + String debugInfo(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java new file mode 100644 index 00000000000..ce59c19ec5a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImpl.java @@ -0,0 +1,187 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.docker.Container; +import com.yahoo.vespa.hosted.node.admin.docker.Docker; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Administers a host (for now only docker hosts) and its nodes (docker containers nodes). + * + * @author stiankri + */ +public class NodeAdminImpl implements NodeAdmin { + private static final Logger logger = Logger.getLogger(NodeAdmin.class.getName()); + + private static final long MIN_AGE_IMAGE_GC_MILLIS = Duration.ofMinutes(15).toMillis(); + + private final Docker docker; + private final Function<HostName, NodeAgent> nodeAgentFactory; + + private final Map<HostName, NodeAgent> nodeAgents = new HashMap<>(); + + private Map<DockerImage, Long> firstTimeEligibleForGC = Collections.emptyMap(); + + /** + * @param docker interface to docker daemon and docker-related tasks + * @param nodeAgentFactory factory for {@link NodeAgent} objects + */ + public NodeAdminImpl(final Docker docker, final Function<HostName, NodeAgent> nodeAgentFactory) { + this.docker = docker; + this.nodeAgentFactory = nodeAgentFactory; + } + + public void setState(final List<ContainerNodeSpec> containersToRun) { + final List<Container> existingContainers = docker.getAllManagedContainers(); + + synchronizeLocalContainerState(containersToRun, existingContainers); + + garbageCollectDockerImages(containersToRun); + } + + public boolean freezeAndCheckIfAllFrozen() { + for (NodeAgent nodeAgent : nodeAgents.values()) { + nodeAgent.execute(NodeAgent.Command.FREEZE); + } + for (NodeAgent nodeAgent : nodeAgents.values()) { + if (nodeAgent.getState() != NodeAgent.State.FROZEN) { + return false; + } + } + return true; + } + + public void unfreeze() { + for (NodeAgent nodeAgent : nodeAgents.values()) { + nodeAgent.execute(NodeAgent.Command.UNFREEZE); + } + } + + public Set<HostName> getListOfHosts() { + return nodeAgents.keySet(); + } + + @Override + public String debugInfo() { + StringBuilder debug = new StringBuilder(); + for (Map.Entry<HostName, NodeAgent> node : nodeAgents.entrySet()) { + debug.append("Node ").append(node.getKey().toString()); + debug.append(" state ").append(node.getValue().getState()); + } + return debug.toString(); + } + + private void garbageCollectDockerImages(final List<ContainerNodeSpec> containersToRun) { + final Set<DockerImage> deletableDockerImages = getDeletableDockerImages( + docker.getUnusedDockerImages(), containersToRun); + final long currentTime = System.currentTimeMillis(); + // TODO: This logic should be unit tested. + firstTimeEligibleForGC = deletableDockerImages.stream() + .collect(Collectors.toMap( + dockerImage -> dockerImage, + dockerImage -> Optional.ofNullable(firstTimeEligibleForGC.get(dockerImage)).orElse(currentTime))); + // Delete images that have been eligible for some time. + firstTimeEligibleForGC.forEach((dockerImage, timestamp) -> { + if (currentTime - timestamp > MIN_AGE_IMAGE_GC_MILLIS) { + docker.deleteImage(dockerImage); + } + }); + } + + // Turns an Optional<T> into a Stream<T> of length zero or one depending upon whether a value is present. + // This is a workaround for Java 8 not having Stream.flatMap(Optional). + private static <T> Stream<T> streamOf(Optional<T> opt) { + return opt.map(Stream::of) + .orElseGet(Stream::empty); + } + + static Set<DockerImage> getDeletableDockerImages( + final Set<DockerImage> currentlyUnusedDockerImages, + final List<ContainerNodeSpec> pendingContainers) { + final Set<DockerImage> imagesNeededNowOrInTheFuture = pendingContainers.stream() + .flatMap(nodeSpec -> streamOf(nodeSpec.wantedDockerImage)) + .collect(Collectors.toSet()); + return diff(currentlyUnusedDockerImages, imagesNeededNowOrInTheFuture); + } + + // Set-difference. Returns minuend minus subtrahend. + private static <T> Set<T> diff(final Set<T> minuend, final Set<T> subtrahend) { + final HashSet<T> result = new HashSet<>(minuend); + result.removeAll(subtrahend); + return result; + } + + // Returns a full outer join of two data sources (of types T and U) on some extractable attribute (of type V). + // Full outer join means that all elements of both data sources are included in the result, + // even when there is no corresponding element (having the same attribute) in the other data set, + // in which case the value from the other source will be empty. + static <T, U, V> Stream<Pair<Optional<T>, Optional<U>>> fullOuterJoin( + final Stream<T> tStream, final Function<T, V> tAttributeExtractor, + final Stream<U> uStream, final Function<U, V> uAttributeExtractor) { + final Map<V, T> tMap = tStream.collect(Collectors.toMap(tAttributeExtractor, t -> t)); + final Map<V, U> uMap = uStream.collect(Collectors.toMap(uAttributeExtractor, u -> u)); + return Stream.concat(tMap.keySet().stream(), uMap.keySet().stream()) + .distinct() + .map(key -> new Pair<>(Optional.ofNullable(tMap.get(key)), Optional.ofNullable(uMap.get(key)))); + } + + void synchronizeLocalContainerState( + final List<ContainerNodeSpec> containersToRun, + final List<Container> existingContainers) { + final Stream<Pair<Optional<ContainerNodeSpec>, Optional<Container>>> nodeSpecContainerPairs = fullOuterJoin( + containersToRun.stream(), nodeSpec -> nodeSpec.hostname, + existingContainers.stream(), container -> container.hostname); + + final Set<HostName> nodeHostNames = containersToRun.stream() + .map(spec -> spec.hostname) + .collect(Collectors.toSet()); + final Set<HostName> obsoleteAgentHostNames = diff(nodeAgents.keySet(), nodeHostNames); + obsoleteAgentHostNames.forEach(hostName -> nodeAgents.remove(hostName).terminate()); + + nodeSpecContainerPairs.forEach(nodeSpecContainerPair -> { + final Optional<ContainerNodeSpec> nodeSpec = nodeSpecContainerPair.getFirst(); + final Optional<Container> existingContainer = nodeSpecContainerPair.getSecond(); + + if (!nodeSpec.isPresent()) { + assert existingContainer.isPresent(); + logger.warning("Container " + existingContainer.get() + " exists, but is not in node repository runlist"); + return; + } + + try { + updateAgent(nodeSpec.get()); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to bring container to desired state", e); + } + }); + } + + private void updateAgent(final ContainerNodeSpec nodeSpec) throws IOException { + final NodeAgent agent; + if (nodeAgents.containsKey(nodeSpec.hostname)) { + agent = nodeAgents.get(nodeSpec.hostname); + } else { + agent = nodeAgentFactory.apply(nodeSpec.hostname); + nodeAgents.put(nodeSpec.hostname, agent); + agent.start(); + } + agent.execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java deleted file mode 100644 index 985e20a3ea8..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; -import com.yahoo.nodeadmin.docker.DockerConfig; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.hosted.node.admin.docker.Docker; -import com.yahoo.vespa.hosted.node.admin.docker.DockerImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; - -import javax.annotation.concurrent.GuardedBy; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static java.util.concurrent.TimeUnit.SECONDS; - -/** - * @author stiankri - */ -public class NodeAdminScheduler extends AbstractComponent { - private static final Logger log = Logger.getLogger(NodeAdminScheduler.class.getName()); - - private static final long INITIAL_DELAY_SECONDS = 0; - private static final long INTERVAL_IN_SECONDS = 60; - - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - private final ScheduledFuture<?> scheduledFuture; - - private enum State { WAIT, WORK, STOP } - - private final Object monitor = new Object(); - @GuardedBy("monitor") - private State state = State.WAIT; - @GuardedBy("monitor") - private List<ContainerNodeSpec> wantedContainerState = null; - - public NodeAdminScheduler(final DockerConfig dockerConfig) { - final Docker docker = new DockerImpl(DockerImpl.newDockerClientFromConfig(dockerConfig)); - final NodeRepository nodeRepository = new NodeRepositoryImpl(); - final Orchestrator orchestrator = new OrchestratorImpl(OrchestratorImpl.makeOrchestratorHostApiClient()); - final Function<HostName, NodeAgent> nodeAgentFactory = (hostName) -> - new NodeAgentImpl(hostName, docker, nodeRepository, orchestrator); - final NodeAdmin nodeAdmin = new NodeAdmin(docker, nodeAgentFactory); - scheduledFuture = scheduler.scheduleWithFixedDelay( - throwableLoggingRunnable(fetchContainersToRunFromNodeRepository(nodeRepository)), - INITIAL_DELAY_SECONDS, INTERVAL_IN_SECONDS, SECONDS); - new Thread(maintainWantedStateRunnable(nodeAdmin), "Node Admin Scheduler main thread").start(); - } - - private void notifyWorkToDo(final Runnable codeToExecuteInCriticalSection) { - synchronized (monitor) { - if (state == State.STOP) { - return; - } - state = State.WORK; - codeToExecuteInCriticalSection.run(); - monitor.notifyAll(); - } - } - - /** - * Prevents exceptions from leaking out and suppressing the scheduler from running the task again. - */ - private static Runnable throwableLoggingRunnable(final Runnable task) { - return () -> { - try { - task.run(); - } catch (Throwable throwable) { - log.log(LogLevel.ERROR, "Unhandled exception leaked out to scheduler.", throwable); - } - }; - } - - private Runnable fetchContainersToRunFromNodeRepository(final NodeRepository nodeRepository) { - return () -> { - // TODO: should the result from the config server contain both active and inactive? - final List<ContainerNodeSpec> containersToRun; - try { - containersToRun = nodeRepository.getContainersToRun(); - } catch (IOException e) { - log.log(Level.WARNING, "Failed fetching container info from node repository", e); - return; - } - setWantedContainerState(containersToRun); - }; - } - - private void setWantedContainerState(final List<ContainerNodeSpec> wantedContainerState) { - if (wantedContainerState == null) { - throw new IllegalArgumentException("wantedContainerState must not be null"); - } - - final Runnable codeToExecuteInCriticalSection = () -> this.wantedContainerState = wantedContainerState; - notifyWorkToDo(codeToExecuteInCriticalSection); - } - - private Runnable maintainWantedStateRunnable(final NodeAdmin nodeAdmin) { - return () -> { - while (true) { - final List<ContainerNodeSpec> containersToRun; - - synchronized (monitor) { - while (state == State.WAIT) { - try { - monitor.wait(); - } catch (InterruptedException e) { - // Ignore, properly handled by next loop iteration. - } - } - if (state == State.STOP) { - return; - } - assert state == State.WORK; - assert wantedContainerState != null; - containersToRun = wantedContainerState; - state = State.WAIT; - } - - throwableLoggingRunnable(() -> nodeAdmin.maintainWantedState(containersToRun)) - .run(); - } - }; - } - - @Override - public void deconstruct() { - scheduledFuture.cancel(false); - scheduler.shutdown(); - synchronized (monitor) { - state = State.STOP; - monitor.notifyAll(); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java new file mode 100644 index 00000000000..97bcdfa724a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdater.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.docker.Docker; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater.State.RESUMED; +import static com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater.State.SUSPENDED; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +/** + * Pulls information from node repository and forwards containers to run to node admin. + * + * @author dybis, stiankri + */ +public class NodeAdminStateUpdater extends AbstractComponent { + private static final Logger log = Logger.getLogger(NodeAdminStateUpdater.class.getName()); + + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + + private final NodeAdmin nodeAdmin; + private boolean isRunningUpdates = true; + private final Object monitor = new Object(); + final Orchestrator orchestrator; + private final String baseHostName; + + public NodeAdminStateUpdater( + final NodeRepository nodeRepository, + final NodeAdmin nodeAdmin, + long initialSchedulerDelayMillis, + long intervalSchedulerInMillis, + Orchestrator orchestrator, + String baseHostName) { + scheduler.scheduleWithFixedDelay( + ()-> fetchContainersToRunFromNodeRepository(nodeRepository), + initialSchedulerDelayMillis, + intervalSchedulerInMillis, + MILLISECONDS); + this.nodeAdmin = nodeAdmin; + this.orchestrator = orchestrator; + this.baseHostName = baseHostName; + } + + public String getDebugPage() { + StringBuilder info = new StringBuilder(); + synchronized (monitor) { + info.append("isRunningUpdates is " + isRunningUpdates+ ". "); + info.append("NodeAdmin: "); + info.append(nodeAdmin.debugInfo()); + } + return info.toString(); + } + + public enum State { RESUMED, SUSPENDED} + + /** + * @return empty on success and failure message on failure. + */ + public Optional<String> setResumeStateAndCheckIfResumed(State wantedState) { + synchronized (monitor) { + isRunningUpdates = wantedState == RESUMED; + } + if (wantedState == SUSPENDED) { + if (! nodeAdmin.freezeAndCheckIfAllFrozen()) { + return Optional.of("Not all node agents are frozen."); + } + } else { + nodeAdmin.unfreeze(); + } + + List<String> hosts = new ArrayList<>(); + nodeAdmin.getListOfHosts().forEach(host -> hosts.add(host.toString())); + if (wantedState == RESUMED) { + return orchestrator.resume(hosts); + } + return orchestrator.suspend(baseHostName, hosts); + } + + private void fetchContainersToRunFromNodeRepository(final NodeRepository nodeRepository) { + synchronized (monitor) { + if (! isRunningUpdates) { + log.log(Level.FINE, "Is frozen, skipping"); + return; + } + final List<ContainerNodeSpec> containersToRun; + try { + containersToRun = nodeRepository.getContainersToRun(); + } catch (Throwable t) { + log.log(Level.WARNING, "Failed fetching container info from node repository", t); + return; + } + if (containersToRun == null) { + log.log(Level.WARNING, "Got null from NodeRepo."); + return; + } + try { + nodeAdmin.setState(containersToRun); + } catch (Throwable t) { + log.log(Level.WARNING, "Failed updating node admin: ", t); + return; + } + } + } + + @Override + public void deconstruct() { + scheduler.shutdown(); + try { + if (! scheduler.awaitTermination(30, TimeUnit.SECONDS)) { + throw new RuntimeException("Did not manage to shutdown scheduler."); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java index 54e7ac3e92f..eac1b228378 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java @@ -2,25 +2,36 @@ package com.yahoo.vespa.hosted.node.admin; /** - * Responsible for management of a single node/container over its lifecycle. + * Responsible for management of a single node over its lifecycle. * May own its own resources, threads etc. Runs independently, but receives signals * on state changes in the environment that may trigger this agent to take actions. * * @author bakksjo */ public interface NodeAgent { + + enum Command {UPDATE_FROM_NODE_REPO, FREEZE, UNFREEZE} + enum State {WAITING, WORKING, DIRTY, FROZEN, TERMINATED} + /** * Signals to the agent that it should update the node specification and container state and maintain wanted state. * - * This method is to be assumed asynchronous by the caller; i.e. any actions the agent will take may execute after this method call returns. + * This method is to be assumed asynchronous by the caller; i.e. any actions the agent will take may execute after + * this method call returns. * * It is an error to call this method on an instance after stop() has been called. */ - void update(); + void execute(Command wantedState); + + /** + * Returns the state of the agent. + */ + State getState(); /** * Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously - * striving to make the current state equal to the wanted state. The current and wanted state update as part of {@link #update()}. + * striving to make the current state equal to the wanted state. The current and wanted state update as part of + * {@link #execute(Command)}. */ void start(); @@ -29,5 +40,5 @@ public interface NodeAgent { * Cleans up any resources the agent owns, such as threads, connections etc. Cleanup is synchronous; when this * method returns, no more actions will be taken by the agent. */ - void stop(); + void terminate(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java index b197b639166..d4af21900c3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -41,11 +42,11 @@ public class NodeAgentImpl implements NodeAgent { private final Thread thread; - private enum State { WAIT, WORK, STOP } - private final Object monitor = new Object(); @GuardedBy("monitor") - private State state = State.WAIT; + private State state = State.WAITING; + @GuardedBy("monitor") + private State wantedState = State.WAITING; // The attributes of the last successful noderepo attribute update for this node. Used to avoid redundant calls. // Only used internally by maintenance thread; no synchronization necessary. @@ -55,6 +56,7 @@ public class NodeAgentImpl implements NodeAgent { + /** * @param hostName the hostname of the node managed by this agent * @param docker interface to docker daemon and docker-related tasks @@ -75,17 +77,34 @@ public class NodeAgentImpl implements NodeAgent { } @Override - public void update() { - changeStateAndNotify(() -> { - this.state = State.WORK; - }); + public void execute(Command command) { + synchronized (monitor) { + switch (command) { + case UPDATE_FROM_NODE_REPO: + wantedState = State.WORKING; + break; + case FREEZE: + wantedState = State.FROZEN; + break; + case UNFREEZE: + wantedState = State.WORKING; + break; + } + } + } + + @Override + public State getState() { + synchronized (monitor) { + return state; + } } @Override public void start() { logger.log(LogLevel.INFO, logPrefix + "Scheduling start of NodeAgent"); synchronized (monitor) { - if (state == State.STOP) { + if (state == State.TERMINATED) { throw new IllegalStateException("Cannot re-start a stopped node agent"); } } @@ -93,14 +112,15 @@ public class NodeAgentImpl implements NodeAgent { } @Override - public void stop() { + public void terminate() { logger.log(LogLevel.INFO, logPrefix + "Scheduling stop of NodeAgent"); - changeStateAndNotify(() -> { - if (state == State.STOP) { + synchronized (monitor) { + if (state == State.TERMINATED) { throw new IllegalStateException("Cannot stop an already stopped node agent"); } - state = State.STOP; - }); + wantedState = State.TERMINATED; + } + monitor.notifyAll(); try { thread.join(); } catch (InterruptedException e) { @@ -360,38 +380,51 @@ public class NodeAgentImpl implements NodeAgent { } private void scheduleWork() { - changeStateAndNotify(() -> state = State.WORK); - } - - private void changeStateAndNotify(final Runnable stateChanger) { synchronized (monitor) { - if (state == State.STOP) { - return; + if (wantedState != State.FROZEN) { + wantedState = State.WORKING; + } else { + logger.log(Level.FINE, "Not scheduling work since in freeze."); } - stateChanger.run(); - monitor.notifyAll(); } + monitor.notifyAll(); } - private void maintainWantedState() { - while (true) { - synchronized (monitor) { - while (state == State.WAIT) { - try { + private void blockUntilNotWaitingOrFrozen() { + try { + synchronized (monitor) { + while (wantedState == State.WAITING || wantedState == State.FROZEN) { + state = wantedState; monitor.wait(); - } catch (InterruptedException e) { - // Ignore, properly handled by next loop iteration. + continue; } } - if (state == State.STOP) { - return; - } - assert state == State.WORK; - state = State.WAIT; + } catch (InterruptedException e) { + throw new RuntimeException(e); } + } + private void maintainWantedState() { + while (true) { + blockUntilNotWaitingOrFrozen(); + synchronized (monitor) { + switch (wantedState) { + case WAITING: + state = State.WAITING; + continue; + case WORKING: + state = State.WORKING; + break; + case FROZEN: + state = State.FROZEN; + continue; + case TERMINATED: + return; + } + } + // This is WORKING state. try { - final ContainerNodeSpec nodeSpec = nodeRepository.getContainer(hostname) + final ContainerNodeSpec nodeSpec = nodeRepository.getContainerNodeSpec(hostname) .orElseThrow(() -> new IllegalStateException(String.format("Node '%s' missing from node repository.", hostname))); final Optional<Container> existingContainer = docker.getContainer(hostname); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java index 34e215da14c..b4cf2fe237f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.docker; import com.google.common.base.Joiner; import com.google.common.io.CharStreams; +import com.google.inject.Inject; import com.spotify.docker.client.ContainerNotFoundException; import com.spotify.docker.client.DefaultDockerClient; import com.spotify.docker.client.DockerCertificateException; @@ -65,8 +66,8 @@ public class DockerImpl implements Docker { private static final int SECONDS_TO_WAIT_BEFORE_KILLING = 10; private static final String FRAMEWORK_CONTAINER_PREFIX = "/"; - private static final String[] COMMAND_YINST_LS_VESPA = new String[]{"yinst", "ls", "vespa"}; - private static final Pattern VESPA_PACKAGE_VERSION_PATTERN = Pattern.compile("^vespa-(\\S+)", Pattern.MULTILINE); + static final String[] COMMAND_GET_VESPA_VERSION = new String[]{"vespa-nodectl", "vespa-version"}; + private static final Pattern VESPA_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+\\S*)", Pattern.MULTILINE); private static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby"; private static final String LABEL_VALUE_MANAGEDBY = "node-admin"; @@ -79,9 +80,7 @@ public class DockerImpl implements Docker { } private static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); - private static final Path RELATIVE_CLEANUP_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage-cleanup"); private static final Path APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN = Paths.get("/host").resolve(RELATIVE_APPLICATION_STORAGE_PATH); - private static final Path CLEANUP_APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN = Paths.get("/host").resolve(RELATIVE_CLEANUP_APPLICATION_STORAGE_PATH); private static final Path APPLICATION_STORAGE_PATH_FOR_HOST = Paths.get("/").resolve(RELATIVE_APPLICATION_STORAGE_PATH); private static final List<String> DIRECTORIES_TO_MOUNT = Arrays.asList( @@ -116,14 +115,16 @@ public class DockerImpl implements Docker { this.docker = dockerClient; } - public static DockerClient newDockerClientFromConfig(final DockerConfig config) { - return DefaultDockerClient.builder(). + @Inject + public DockerImpl(final DockerConfig config) { + this(DefaultDockerClient.builder(). uri(config.uri()). dockerCertificates(certificates(config)). readTimeoutMillis(TimeUnit.MINUTES.toMillis(30)). // Some operations may take minutes. - build(); + build()); } + private static DockerCertificates certificates(DockerConfig config) { try { return DockerCertificates.builder() @@ -204,7 +205,7 @@ public class DockerImpl implements Docker { log.log(LogLevel.INFO, "The application storage at " + from + " doesn't exist"); return; } - Path to = CLEANUP_APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN.resolve(containerName.asString() + "_" + filenameFormatter + Path to = applicationStoragePathForNodeAdmin("cleanup_" + containerName.asString() + "_" + filenameFormatter .format(Date.from(Instant.now()))); log.log(LogLevel.INFO, "Deleting application storage by moving it from " + from + " to " + to); Files.move(from, to); @@ -231,7 +232,7 @@ public class DockerImpl implements Docker { .networkMode("none") .binds(applicationStorageToMount(containerName.asString())) .build()) - .env("CONFIG_SERVER_ADDRESS=" + Joiner.on(',').join(Environment.getConfigServerHostsFromYinstSetting())). + .env("CONFIG_SERVER_ADDRESS=" + Joiner.on(',').join(Environment.getConfigServerHosts())). hostname(hostName.s()); if (minMainMemoryAvailableGb > 0.00001) { containerConfigBuilder.memory((long) (GIGA * minMainMemoryAvailableGb)); @@ -258,16 +259,21 @@ public class DockerImpl implements Docker { @Override public String getVespaVersion(final ContainerName containerName) { - ProcessResult result = executeInContainer(containerName, COMMAND_YINST_LS_VESPA); + ProcessResult result = executeInContainer(containerName, COMMAND_GET_VESPA_VERSION); if (!result.isSuccess()) { throw new RuntimeException("Container " + containerName.asString() + ": Command " - + Arrays.toString(COMMAND_YINST_LS_VESPA) + " failed: " + result); + + Arrays.toString(COMMAND_GET_VESPA_VERSION) + " failed: " + result); } - return parseVespaVersion(result.getOutput()) - .orElseThrow(() -> new RuntimeException( - "Container " + containerName.asString() + ": Failed to parse vespa version from " - + result.getOutput())); + .orElseThrow(() -> new RuntimeException( + "Container " + containerName.asString() + ": Failed to parse vespa version from " + + result.getOutput())); + } + + // Returns empty if vespa version cannot be parsed. + static Optional<String> parseVespaVersion(final String rawVespaVersion) { + final Matcher matcher = VESPA_VERSION_PATTERN.matcher(rawVespaVersion); + return matcher.find() ? Optional.of(matcher.group(1)) : Optional.empty(); } @Override @@ -297,12 +303,6 @@ public class DockerImpl implements Docker { } } - // Returns empty if vespa version cannot be parsed. - static Optional<String> parseVespaVersion(final String outputFromYinstLsVespa) { - final Matcher matcher = VESPA_PACKAGE_VERSION_PATTERN.matcher(outputFromYinstLsVespa); - return matcher.find() ? Optional.of(matcher.group(1)) : Optional.empty(); - } - private void setupContainerNetworking(ContainerName containerName, HostName hostName, int containerPid) throws UnknownHostException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java index 75d6f641feb..41312c28df7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.docker; -import java.io.IOException; import java.util.Objects; public class ProcessResult { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java index 04d68269144..8829c3d4487 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java @@ -15,7 +15,7 @@ import java.util.Optional; public interface NodeRepository { List<ContainerNodeSpec> getContainersToRun() throws IOException; - Optional<ContainerNodeSpec> getContainer(HostName hostname) throws IOException; + Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException; void updateNodeAttributes( HostName hostName, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java index 87b5ecd3a93..bc159c89781 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java @@ -29,40 +29,18 @@ import java.util.logging.Logger; */ public class NodeRepositoryImpl implements NodeRepository { private static final Logger logger = Logger.getLogger(NodeRepositoryImpl.class.getName()); - private static final int HARDCODED_NODEREPOSITORY_PORT = 19071; private static final String NODEREPOSITORY_PATH_PREFIX_NODES_API = "/"; - private static final String ENV_HOSTNAME = "HOSTNAME"; private JaxRsStrategy<NodeRepositoryApi> nodeRepositoryClient; private final String baseHostName; - public NodeRepositoryImpl() { - baseHostName = Optional.ofNullable(System.getenv(ENV_HOSTNAME)) - .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset")); - nodeRepositoryClient = getApi(); - } - - // For testing - NodeRepositoryImpl(String baseHostName, String configserver, int configport) { - this.baseHostName = baseHostName; - final Set<HostName> configServerHosts = new HashSet<>(); - configServerHosts.add(new HostName(configserver)); - - final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory(); - final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory( - configServerHosts, configport, jaxRsClientFactory); - nodeRepositoryClient = jaxRsStrategyFactory.apiWithRetries(NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API); - } - - private static JaxRsStrategy<NodeRepositoryApi> getApi() { - final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting(); - if (configServerHosts.isEmpty()) { - throw new IllegalStateException("Environment setting for config servers missing or empty."); - } + public NodeRepositoryImpl(Set<HostName> configServerHosts, int configPort, String baseHostName) { final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory(); final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory( - configServerHosts, HARDCODED_NODEREPOSITORY_PORT, jaxRsClientFactory); - return jaxRsStrategyFactory.apiWithRetries(NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API); + configServerHosts, configPort, jaxRsClientFactory); + this.nodeRepositoryClient = jaxRsStrategyFactory.apiWithRetries( + NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API); + this.baseHostName = baseHostName; } @Override @@ -92,11 +70,15 @@ public class NodeRepositoryImpl implements NodeRepository { } @Override - public Optional<ContainerNodeSpec> getContainer(HostName hostname) throws IOException { - // TODO Use proper call to node repository - return getContainersToRun().stream() - .filter(cns -> Objects.equals(hostname, cns.hostname)) - .findFirst(); + public Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException { + final GetNodesResponse response = nodeRepositoryClient.apply(nodeRepositoryApi -> nodeRepositoryApi.getNode(hostName.toString(), true)); + if (response.nodes.size() == 0) { + return Optional.empty(); + } + if (response.nodes.size() != 1) { + throw new RuntimeException("Did not get data for one node using hostname=" + hostName.toString() + "\n" + response.toString()); + } + return Optional.of(createContainerNodeSpec(response.nodes.get(0))); } private static ContainerNodeSpec createContainerNodeSpec(GetNodesResponse.Node node) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java index 36ab89a6718..519cc7ec3d3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java @@ -21,6 +21,18 @@ public interface NodeRepositoryApi { @QueryParam("parentHost") String hostname, @QueryParam("recursive") boolean recursive); + /** + * What is called "host" in NodeRepo is called "node" in node admin in this case. + * @param node the node to get data about. + * @param recursive set this to true, or you will not get the data you expect. + * @return + */ + @GET + @Path("/nodes/v2/node/") + GetNodesResponse getNode( + @QueryParam("hostname") String node, + @QueryParam("recursive") boolean recursive); + @PUT @Path("/nodes/v2/state/ready/{hostname}") // TODO: remove fake return String body; should be void and empty diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java index 00365b5fc3d..3afa37075a2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.node.admin.orchestrator; import com.yahoo.vespa.applicationmodel.HostName; +import java.util.List; +import java.util.Optional; + /** * Abstraction for communicating with Orchestrator. * @@ -18,4 +21,14 @@ public interface Orchestrator { * Invokes orchestrator resume of a host. Returns whether resume was granted. */ boolean resume(HostName hostName); + + /** + * Invokes orchestrator suspend hosts. Returns failure reasons when failing. + */ + Optional<String> suspend(String parentHostName, List<String> hostNames); + + /** + * Invokes orchestrator resume of hosts. Returns failure reasons when failing. + */ + Optional<String> resume(List<String> hostName); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java index 422962bdc58..804c06dccae 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java @@ -7,6 +7,9 @@ import com.yahoo.vespa.jaxrs.client.JaxRsStrategy; import com.yahoo.vespa.jaxrs.client.JaxRsStrategyFactory; import com.yahoo.vespa.jaxrs.client.JerseyJaxRsClientFactory; import com.yahoo.vespa.orchestrator.restapi.HostApi; +import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; import com.yahoo.vespa.applicationmodel.HostName; @@ -14,7 +17,8 @@ import javax.ws.rs.ClientErrorException; import javax.ws.rs.NotFoundException; import javax.ws.rs.core.Response; import java.io.IOException; -import java.util.HashSet; +import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -22,6 +26,7 @@ import java.util.logging.Logger; /** * @author stiankri * @author bakksjo + * @author dybis */ public class OrchestratorImpl implements Orchestrator { private static final Logger logger = Logger.getLogger(OrchestratorImpl.class.getName()); @@ -32,19 +37,21 @@ public class OrchestratorImpl implements Orchestrator { private static final String ORCHESTRATOR_PATH_PREFIX_HOST_API = ORCHESTRATOR_PATH_PREFIX + HostApi.PATH_PREFIX; - // We use this to allow client code to treat resume() calls as idempotent and cheap, - // but we actually filter out redundant resume calls to orchestrator. - private final Set<HostName> resumedHosts = new HashSet<>(); + private static final String ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API + = ORCHESTRATOR_PATH_PREFIX + HostSuspensionApi.PATH_PREFIX; + private final JaxRsStrategy<HostApi> hostApiClient; + private final JaxRsStrategy<HostSuspensionApi> hostSuspensionClient; + - public OrchestratorImpl(JaxRsStrategy<HostApi> hostApiClient) { + public OrchestratorImpl(JaxRsStrategy<HostApi> hostApiClient, JaxRsStrategy<HostSuspensionApi> hostSuspensionClient) { this.hostApiClient = hostApiClient; + this.hostSuspensionClient = hostSuspensionClient; } @Override public boolean suspend(final HostName hostName) { - resumedHosts.remove(hostName); try { return hostApiClient.apply(api -> { final UpdateHostResponse response = api.suspend(hostName.s()); @@ -64,19 +71,33 @@ public class OrchestratorImpl implements Orchestrator { } @Override - public boolean resume(final HostName hostName) { - if (resumedHosts.contains(hostName)) { - return true; + public Optional<String> suspend(String parentHostName, List<String> hostNames) { + try { + return hostSuspensionClient.apply(hostSuspensionClient -> { + BatchHostSuspendRequest request = new BatchHostSuspendRequest(parentHostName, hostNames); + final BatchOperationResult result = hostSuspensionClient.suspendAll(request); + return result.getFailureReason(); + }); + } catch (ClientErrorException e) { + if (e instanceof NotFoundException || e.getResponse().getStatus() == Response.Status.NOT_FOUND.getStatusCode()) { + // Orchestrator doesn't care about this node, so don't let that stop us. + return Optional.empty(); + } + logger.log(Level.INFO, "Orchestrator rejected suspend request for host " + parentHostName, e); + return Optional.of(e.getLocalizedMessage()); + } catch (IOException e) { + logger.log(Level.WARNING, "Unable to communicate with orchestrator", e); + return Optional.of("Unable to communicate with orchestrator" + e.getMessage()); } + } + @Override + public boolean resume(final HostName hostName) { try { final boolean resumeSucceeded = hostApiClient.apply(api -> { final UpdateHostResponse response = api.resume(hostName.s()); return response.reason() == null; }); - if (resumeSucceeded) { - resumedHosts.add(hostName); - } return resumeSucceeded; } catch (ClientErrorException e) { logger.log(Level.INFO, "Orchestrator rejected resume request for host " + hostName, e); @@ -87,15 +108,26 @@ public class OrchestratorImpl implements Orchestrator { } } - public static JaxRsStrategy<HostApi> makeOrchestratorHostApiClient() { - final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting(); + @Override + public Optional<String> resume(List<String> hostNames) { + for (String host : hostNames) { + if (! resume(new HostName(host))) { + return Optional.of("Could not resume " + host); + } + } + return Optional.empty(); + } + + public static OrchestratorImpl createOrchestratorFromSettings() { + final Set<HostName> configServerHosts = Environment.getConfigServerHosts(); if (configServerHosts.isEmpty()) { - throw new IllegalStateException("Emnvironment setting for config servers missing or empty."); + throw new IllegalStateException("Environment setting for config servers missing or empty."); } final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory(); final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory( configServerHosts, HARDCODED_ORCHESTRATOR_PORT, jaxRsClientFactory); - return jaxRsStrategyFactory.apiWithRetries(HostApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_API); + JaxRsStrategy<HostApi> hostApi = jaxRsStrategyFactory.apiWithRetries(HostApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_API); + JaxRsStrategy<HostSuspensionApi> suspendApi = jaxRsStrategyFactory.apiWithRetries(HostSuspensionApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API); + return new OrchestratorImpl(hostApi, suspendApi); } - } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java new file mode 100644 index 00000000000..343ac24b20f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java @@ -0,0 +1,13 @@ +package com.yahoo.vespa.hosted.node.admin.provider; + +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; + +/** + * Class for setting up instances of classes; enables testing. + * + * @author dybis + */ +public interface ComponentsProvider { + NodeAdminStateUpdater getNodeAdminStateUpdater(); + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java new file mode 100644 index 00000000000..1bdc59f22ea --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.hosted.node.admin.provider; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.NodeAdmin; +import com.yahoo.vespa.hosted.node.admin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.docker.Docker; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.util.Set; +import java.util.function.Function; + +/** + * Set up node admin for production. + * + * @author dybis + */ +public class ComponentsProviderImpl implements ComponentsProvider { + + private final Docker docker; + private static final long INITIAL_SCHEDULER_DELAY_SECONDS = 0; + private static final long INTERVAL_SCHEDULER_IN_SECONDS = 60; + + private static final int HARDCODED_NODEREPOSITORY_PORT = 19071; + private static final String ENV_HOSTNAME = "HOSTNAME"; + public ComponentsProviderImpl(final Docker docker) { + this.docker = docker; + } + + @Override + public NodeAdminStateUpdater getNodeAdminStateUpdater() { + String baseHostName = java.util.Optional.ofNullable(System.getenv(ENV_HOSTNAME)) + .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset")); + + Set<HostName> configServerHosts = Environment.getConfigServerHosts(); + if (configServerHosts.isEmpty()) { + throw new IllegalStateException("Environment setting for config servers missing or empty."); + } + + NodeRepository nodeRepository = new NodeRepositoryImpl(configServerHosts, HARDCODED_NODEREPOSITORY_PORT, baseHostName); + + Orchestrator orchestrator = OrchestratorImpl.createOrchestratorFromSettings(); + final Function<HostName, NodeAgent> nodeAgentFactory = (hostName) -> + new NodeAgentImpl(hostName, docker, nodeRepository, orchestrator); + final NodeAdmin nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory); + return new NodeAdminStateUpdater( + nodeRepository, nodeAdmin, INITIAL_SCHEDULER_DELAY_SECONDS, INTERVAL_SCHEDULER_IN_SECONDS, orchestrator, baseHostName); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java new file mode 100644 index 00000000000..468577e8f4f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.provider; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java deleted file mode 100644 index 9c2af68d56a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.restapi; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; - -/** - * @author stiankri - */ -@Path("") -public interface NodeAdminRestAPI { - @GET - @Path("/update") - public String update(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java new file mode 100644 index 00000000000..bab9c521024 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java @@ -0,0 +1,95 @@ +package com.yahoo.vespa.hosted.node.admin.restapi; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; + +import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProvider; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; + +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; + +/** + * Rest API for suspending and resuming the docker host. + * There are two non-blocking idempotent calls: /resume and /suspend. + * + * There is one debug call: /info + * + * @author dybis + */ +public class RestApiHandler extends LoggingRequestHandler{ + + private final NodeAdminStateUpdater refresher; + + public RestApiHandler(Executor executor, AccessLog accessLog, ComponentsProvider componentsProvider) { + super(executor, accessLog); + this.refresher = componentsProvider.getNodeAdminStateUpdater(); + } + + @Override + public HttpResponse handle(HttpRequest request) { + if (request.getMethod() == GET) { + return handleGet(request); + } + if (request.getMethod() == PUT) { + return handlePut(request); + } + return new SimpleResponse(400, "Only PUT and GET are implemented."); + + } + + private HttpResponse handleGet(HttpRequest request) { + return new SimpleResponse(200, refresher.getDebugPage()); + } + + private HttpResponse handlePut(HttpRequest request) { + String path = request.getUri().getPath(); + // Check paths to disallow illegal state changes + if (path.endsWith("resume")) { + final Optional<String> resumed = refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED); + if (resumed.isPresent()) { + return new SimpleResponse(400, resumed.get()); + } + return new SimpleResponse(200, "ok."); + } + if (path.endsWith("suspend")) { + Optional<String> resumed = refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); + if (resumed.isPresent()) { + return new SimpleResponse(423, resumed.get()); + } + return new SimpleResponse(200, "ok"); + } + return new SimpleResponse(400, "unknown path" + path); + } + + private static class SimpleResponse extends HttpResponse { + + private final String jsonMessage; + + public SimpleResponse(int code, String message) { + super(code); + // TODO: Use some library to build json as this easily fails + this.jsonMessage = "{ \"jsonMessage\":\"" + message + "\"}"; + } + + @Override + public String getContentType() { + return MediaType.APPLICATION_JSON; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(jsonMessage.getBytes(StandardCharsets.UTF_8.name())); + } + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java new file mode 100644 index 00000000000..5e5bd002950 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.restapi; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java deleted file mode 100644 index 65da2004854..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.testapi; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -/** - * Resource for use in integration test, will be deleted soon. - * @author tonytv - */ -@Path("ping") -public class PingResource { - @GET - @Produces(MediaType.TEXT_PLAIN) - public String ping() { - return "pong"; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java index d6703326796..b5929653458 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java @@ -22,13 +22,13 @@ public class Environment { public enum NetworkType { normal, local, vm } - public static Set<HostName> getConfigServerHostsFromYinstSetting() { - final String yinstSetting = System.getenv(ENV_CONFIGSERVERS); - if (yinstSetting == null) { + public static Set<HostName> getConfigServerHosts() { + final String configServerHosts = System.getenv(ENV_CONFIGSERVERS); + if (configServerHosts == null) { return Collections.emptySet(); } - final List<String> hostNameStrings = Arrays.asList(yinstSetting.split("[,\\s]+")); + final List<String> hostNameStrings = Arrays.asList(configServerHosts.split("[,\\s]+")); return hostNameStrings.stream() .map(HostName::new) .collect(Collectors.toSet()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImplTest.java index a4843a0753e..f3bfeb47e9f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminImplTest.java @@ -34,7 +34,7 @@ import static org.mockito.Mockito.when; /** * @author bakksjo */ -public class NodeAdminTest { +public class NodeAdminImplTest { private static final Optional<Double> MIN_CPU_CORES = Optional.of(1.0); private static final Optional<Double> MIN_MAIN_MEMORY_AVAILABLE_GB = Optional.of(1.0); private static final Optional<Double> MIN_DISK_AVAILABLE_GB = Optional.of(1.0); @@ -47,7 +47,7 @@ public class NodeAdminTest { final Docker docker = mock(Docker.class); final Function<HostName, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); - final NodeAdmin nodeAdmin = new NodeAdmin(docker, nodeAgentFactory); + final NodeAdminImpl nodeAdmin = new NodeAdminImpl(docker, nodeAgentFactory); final NodeAgent nodeAgent1 = mock(NodeAgentImpl.class); final NodeAgent nodeAgent2 = mock(NodeAgentImpl.class); @@ -76,31 +76,31 @@ public class NodeAdminTest { nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer)); inOrder.verify(nodeAgentFactory).apply(hostName); inOrder.verify(nodeAgent1).start(); - inOrder.verify(nodeAgent1).update(); - inOrder.verify(nodeAgent1, never()).stop(); + inOrder.verify(nodeAgent1).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); + inOrder.verify(nodeAgent1, never()).terminate(); nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer)); inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class)); inOrder.verify(nodeAgent1, never()).start(); - inOrder.verify(nodeAgent1).update(); - inOrder.verify(nodeAgent1, never()).stop(); + inOrder.verify(nodeAgent1).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); + inOrder.verify(nodeAgent1, never()).terminate(); nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), asList(existingContainer)); inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class)); - inOrder.verify(nodeAgent1, never()).update(); - verify(nodeAgent1).stop(); + inOrder.verify(nodeAgent1, never()).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); + verify(nodeAgent1).terminate(); nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer)); inOrder.verify(nodeAgentFactory).apply(hostName); inOrder.verify(nodeAgent2).start(); - inOrder.verify(nodeAgent2).update(); - inOrder.verify(nodeAgent2, never()).stop(); + inOrder.verify(nodeAgent2).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); + inOrder.verify(nodeAgent2, never()).terminate(); nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), Collections.emptyList()); inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class)); inOrder.verify(nodeAgent2, never()).start(); - inOrder.verify(nodeAgent2, never()).update(); - inOrder.verify(nodeAgent2).stop(); + inOrder.verify(nodeAgent2, never()).execute(NodeAgent.Command.UPDATE_FROM_NODE_REPO); + inOrder.verify(nodeAgent2).terminate(); verifyNoMoreInteractions(nodeAgent1); verifyNoMoreInteractions(nodeAgent2); @@ -116,7 +116,7 @@ public class NodeAdminTest { final Set<DockerImage> currentlyUnusedImages = Collections.emptySet(); final List<ContainerNodeSpec> pendingContainers = Collections.emptyList(); - final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); + final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); assertThat(deletableImages, is(Collections.emptySet())); } @@ -127,7 +127,7 @@ public class NodeAdminTest { .collect(Collectors.toSet()); final List<ContainerNodeSpec> pendingContainers = Collections.emptyList(); - final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); + final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); final Set<DockerImage> expectedDeletableImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3) .collect(Collectors.toSet()); @@ -139,10 +139,10 @@ public class NodeAdminTest { final Set<DockerImage> currentlyUnusedImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3) .collect(Collectors.toSet()); final List<ContainerNodeSpec> pendingContainers = Stream.of(IMAGE_2, IMAGE_4) - .map(NodeAdminTest::newNodeSpec) + .map(NodeAdminImplTest::newNodeSpec) .collect(Collectors.toList()); - final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); + final Set<DockerImage> deletableImages = NodeAdminImpl.getDeletableDockerImages(currentlyUnusedImages, pendingContainers); final Set<DockerImage> expectedDeletableImages = Stream.of(IMAGE_1, IMAGE_3) .collect(Collectors.toSet()); @@ -168,7 +168,7 @@ public class NodeAdminTest { newPair(null, 21))); assertThat( - NodeAdmin.fullOuterJoin( + NodeAdminImpl.fullOuterJoin( strings.stream(), string -> string, integers.stream(), String::valueOf) .collect(Collectors.toSet()), diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java new file mode 100644 index 00000000000..f9d9ffdf0be --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminStateUpdaterTest.java @@ -0,0 +1,85 @@ +package com.yahoo.vespa.hosted.node.admin; + +import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; +import com.yahoo.vespa.hosted.node.admin.integrationTests.OrchestratorMock; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import org.junit.Test; +import org.mockito.stubbing.Answer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; + +import static junit.framework.TestCase.assertTrue; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.junit.MatcherAssert.assertThat; +import static org.mockito.Matchers.anyList; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Basic test of ActiveContainersRefresherTest + * @author dybis + */ +public class NodeAdminStateUpdaterTest { + + @Test + @SuppressWarnings("unchecked") + public void testExceptionIsCaughtAndDataIsPassedAndFreeze() throws Exception { + NodeRepository nodeRepository = mock(NodeRepository.class); + NodeAdmin nodeAdmin = mock(NodeAdmin.class); + final List<ContainerNodeSpec> accumulatedArgumentList = Collections.synchronizedList(new ArrayList<>()); + final CountDownLatch latch = new CountDownLatch(5); + doAnswer((Answer<Object>) invocation -> { + List<ContainerNodeSpec> containersToRunInArgument = (List<ContainerNodeSpec>) invocation.getArguments()[0]; + containersToRunInArgument.forEach(element -> accumulatedArgumentList.add(element)); + latch.countDown(); + if (accumulatedArgumentList.size() == 2) { + throw new RuleBaseException("This exception is expected, and should show up in the log."); + } + return null; + }).when(nodeAdmin).setState(anyList()); + + final List<ContainerNodeSpec> containersToRun = new ArrayList<>(); + containersToRun.add(createSample()); + + when(nodeRepository.getContainersToRun()).thenReturn(containersToRun); + OrchestratorMock orchestratorMock = new OrchestratorMock(); + NodeAdminStateUpdater refresher = new NodeAdminStateUpdater( + nodeRepository, nodeAdmin, 1, 1, orchestratorMock, "basehostname"); + latch.await(); + int numberOfElements = accumulatedArgumentList.size(); + assertThat(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), + is(Optional.of("Not all node agents are frozen."))); + assertTrue(numberOfElements > 4); + assertThat(accumulatedArgumentList.get(0), is(createSample())); + Thread.sleep(2); + assertThat(accumulatedArgumentList.size(), is(numberOfElements)); + assertThat(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED), + is(Optional.empty())); + while (accumulatedArgumentList.size() == numberOfElements) { + Thread.sleep(1); + } + refresher.deconstruct(); + } + + private ContainerNodeSpec createSample() { + return new ContainerNodeSpec( + new HostName("hostname"), + Optional.empty(), + new ContainerName("containername"), + NodeState.ACTIVE, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java index 9351e80cdab..cc297d8eef7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java @@ -87,12 +87,18 @@ public class DockerImplTest { @Test public void vespaVersionIsParsed() { - assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53"), is(Optional.of("5.119.53"))); + assertThat(DockerImpl.parseVespaVersion("5.119.53"), is(Optional.of("5.119.53"))); } @Test public void vespaVersionIsParsedWithTrailingNewline() { - assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53\n"), is(Optional.of("5.119.53"))); + assertThat(DockerImpl.parseVespaVersion("5.119.53\n"), is(Optional.of("5.119.53"))); + } + + @Test + public void vespaVersionIsParsedWithIrregularVersionScheme() { + assertThat(DockerImpl.parseVespaVersion("7.2"), is(Optional.of("7.2"))); + assertThat(DockerImpl.parseVespaVersion("8.0-beta"), is(Optional.of("8.0-beta"))); } @Test(expected = NullPointerException.class) @@ -107,11 +113,12 @@ public class DockerImplTest { @Test public void vespaVersionIsNotParsedFromUnexpectedContent() { - assertThat(DockerImpl.parseVespaVersion("honda-5.119.53"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("vespa 5.119.53"), is(Optional.empty())); + assertThat(DockerImpl.parseVespaVersion("foo"), is(Optional.empty())); + assertThat(DockerImpl.parseVespaVersion("119"), is(Optional.empty())); + assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53"), is(Optional.empty())); assertThat(DockerImpl.parseVespaVersion("vespa- 5.119.53"), is(Optional.empty())); assertThat(DockerImpl.parseVespaVersion("vespa-"), is(Optional.empty())); - assertThat(DockerImpl.parseVespaVersion("No such command 'yinst'"), is(Optional.empty())); + assertThat(DockerImpl.parseVespaVersion("No such command 'vespanodectl'"), is(Optional.empty())); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java new file mode 100644 index 00000000000..cf147ce7936 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java @@ -0,0 +1,20 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProvider; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; + +/** + * For setting up test with mocks. + * + * @author dybis + */ +public class ComponentsProviderWithMocks implements ComponentsProvider { + private NodeRepoMock nodeRepositoryMock = new NodeRepoMock(); + private NodeAdminMock nodeAdminMock = new NodeAdminMock(); + private OrchestratorMock orchestratorMock = new OrchestratorMock(); + + @Override + public NodeAdminStateUpdater getNodeAdminStateUpdater() { + return new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdminMock, 1, 5, orchestratorMock, "hostname"); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java new file mode 100644 index 00000000000..4043195ea5e --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminMock.java @@ -0,0 +1,64 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.NodeAdmin; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Mock with some simple logic + * + * @author dybis + */ +public class NodeAdminMock implements NodeAdmin { + + StringBuilder info = new StringBuilder(); + + Set<HostName> hostnames = new HashSet<>(); + + boolean freezeSetState = false; + public AtomicBoolean frozen = new AtomicBoolean(false); + + // We make it threadsafe as the test have its own peeking thread. + private Object monitor = new Object(); + + @Override + public void setState(List<ContainerNodeSpec> containersToRun) { + synchronized (monitor) { + hostnames.clear(); + containersToRun.forEach(container -> hostnames.add(container.hostname)); + } + } + + @Override + public boolean freezeAndCheckIfAllFrozen() { + info.append(" Freeze called while in state " + frozen.get()); + freezeSetState = true; + return frozen.get(); + } + + @Override + public void unfreeze() { + info.append(" Unfreeze called while in state " + frozen.get()); + freezeSetState = false; + } + + @Override + public Set<HostName> getListOfHosts() { + synchronized (monitor) { + return hostnames; + } + } + + /* + * We use this to get some information easily out of the mock in the integration test here. + */ + @Override + public String debugInfo() { + return info.toString(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java new file mode 100644 index 00000000000..1d4bb297e49 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Mock with some simple logic + * @author dybis + */ +public class NodeRepoMock implements NodeRepository { + + public List<ContainerNodeSpec> containerNodeSpecs = new ArrayList<>(); + + @Override + public List<ContainerNodeSpec> getContainersToRun() throws IOException { + return containerNodeSpecs; + } + + @Override + public Optional<ContainerNodeSpec> getContainerNodeSpec(HostName hostName) throws IOException { + return null; + } + + @Override + public void updateNodeAttributes(HostName hostName, long restartGeneration, DockerImage dockerImage, String containerVespaVersion) throws IOException { + + } + + @Override + public void markAsReady(HostName hostName) throws IOException { + + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java new file mode 100644 index 00000000000..b9abce88f38 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java @@ -0,0 +1,36 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; + +import java.util.List; +import java.util.Optional; + +/** + * Mock with some simple logic + * @author dybis + */ +public class OrchestratorMock implements Orchestrator { + + public Optional<String> suspendReturnValue = Optional.empty(); + + @Override + public boolean suspend(HostName hostName) { + return false; + } + + @Override + public boolean resume(HostName hostName) { + return false; + } + + @Override + public Optional<String> suspend(String parentHostName, List<String> hostNames) { + return suspendReturnValue; + } + + @Override + public Optional<String> resume(List<String> hostName) { + return Optional.empty(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java new file mode 100644 index 00000000000..e9d367a3f59 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ResumeTest.java @@ -0,0 +1,76 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; +import org.junit.Test; + +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * Scenario test for NodeAdminStateUpdater. + * @author dybis + */ +public class ResumeTest { + @Test + public void test() throws InterruptedException { + NodeRepoMock nodeRepositoryMock = new NodeRepoMock(); + NodeAdminMock nodeAdminMock = new NodeAdminMock(); + OrchestratorMock orchestratorMock = new OrchestratorMock(); + + nodeRepositoryMock.containerNodeSpecs.add(new ContainerNodeSpec( + new HostName("hostname"), + Optional.of(new DockerImage("dockerimage")), + new ContainerName("containe"), + NodeState.ACTIVE, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty())); + + NodeAdminStateUpdater updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdminMock, 1, 1, orchestratorMock, "basehostname"); + // Wait for node admin to be notified with node repo state + while (nodeAdminMock.getListOfHosts().size() == 0) { + Thread.sleep(1); + } + + // Make node admin and orchestrator block suspend + orchestratorMock.suspendReturnValue = Optional.of("orch reject suspend"); + nodeAdminMock.frozen.set(false); + assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.of("Not all node agents are frozen."))); + + // Now, change data in node repo, should not propagate. + nodeRepositoryMock.containerNodeSpecs.clear(); + + // Set node admin not blocking for suspend + nodeAdminMock.frozen.set(true); + assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.of("orch reject suspend"))); + + // Make orchestrator allow suspend + orchestratorMock.suspendReturnValue = Optional.empty(); + assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.empty())); + + // Now suspended, new node repo state should have not propagated to node admin + Thread.sleep(2); + assertThat(nodeAdminMock.getListOfHosts().size(), is(1)); + + // Now resume + nodeAdminMock.frozen.set(false); + assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED), is(Optional.empty())); + + // Now node repo state should propagate to node admin again + while (nodeAdminMock.getListOfHosts().size() != 0) { + Thread.sleep(1); + } + + updater.deconstruct(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java new file mode 100644 index 00000000000..b704e4624b7 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -0,0 +1,129 @@ +package com.yahoo.vespa.hosted.node.admin.integrationTests; + +import com.google.common.collect.Sets; +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; +import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState; +import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.impl.client.HttpClientBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringWriter; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author dybis + */ +public class RunInContainerTest { + + private JDisc container; + private int port; + + private int findRandomOpenPort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + @Before + public void startContainer() throws Exception { + port = findRandomOpenPort(); + System.out.println("PORT IS " + port); + container = JDisc.fromServicesXml(createServiceXml(port), Networking.enable); + } + + private boolean doPutCall(String command) throws IOException { + HttpClient httpclient = HttpClientBuilder.create().build(); + HttpHost target = new HttpHost("localhost", port, "http"); + HttpPut getRequest = new HttpPut("/rest/" + command); + HttpResponse httpResponse = httpclient.execute(target, getRequest); + return httpResponse.getStatusLine().getStatusCode() == 200; + } + + private String doGetInfoCall() throws IOException { + HttpClient httpclient = HttpClientBuilder.create().build(); + HttpHost target = new HttpHost("localhost", port, "http"); + HttpGet getRequest = new HttpGet("/rest/info"); + HttpResponse httpResponse = httpclient.execute(target, getRequest); + HttpEntity entity = httpResponse.getEntity(); + StringWriter writer = new StringWriter(); + IOUtils.copy(entity.getContent(), writer, StandardCharsets.UTF_8); + return writer.toString(); + } + + private void waitForJdiscContainerToServe() throws InterruptedException { + Instant start = Instant.now(); + while (Instant.now().minusSeconds(120).isBefore(start)) { + try { + HttpClient httpclient = HttpClientBuilder.create().build(); + HttpHost target = new HttpHost("localhost", port, "http"); + HttpGet getRequest = new HttpGet("/rest/info"); + HttpResponse httpResponse = httpclient.execute(target, getRequest); + HttpEntity entity = httpResponse.getEntity(); + if (httpResponse.getStatusLine().getStatusCode() != 200) { + continue; + } + System.out.println("Container started."); + return; + } catch (Exception e) { + Thread.sleep(100); + } + } + throw new RuntimeException("Could not get answer from container."); + } + + @After + public void stopContainer() { + if (container != null) { + container.close(); + } + } + + @Test + public void testGetContainersToRunAPi() throws IOException, InterruptedException { + waitForJdiscContainerToServe(); + assertThat(doPutCall("resume"), is(true)); + assertThat(doPutCall("suspend"), is(false)); + assertThat(doGetInfoCall(), is("{ \"jsonMessage\":\"isRunningUpdates is false. NodeAdmin: " + + "Unfreeze called while in state false " + + "Freeze called while in state false\"}")); + } + + + private String createServiceXml(int port) { + return "<services version=\"1.0\">\n" + + " <jdisc version=\"1.0\" jetty=\"true\">\n" + + " <handler id=\"com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler\" bundle=\"node-admin\">\n" + + " <binding>http://*/rest/*</binding>\n" + + " </handler>\n" + + " <component id=\"node-admin\" class=\"com.yahoo.vespa.hosted.node.admin.integrationTests.ComponentsProviderWithMocks\" bundle=\"node-admin\"/>\n" + + " <http>" + + " <server id=\'myServer\' port=\'" + port + "\' />" + + " </http>" + + " </jdisc>\n" + + "</services>\n"; + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java index 633405cc5f7..f2cca110b61 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.vespa.hosted.node.admin.noderepository; +import com.google.common.collect.Sets; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; import com.yahoo.vespa.applicationmodel.HostName; @@ -8,6 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.ContainerName; import com.yahoo.vespa.hosted.node.admin.docker.DockerImage; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; + import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -17,14 +20,22 @@ import java.net.ServerSocket; import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; - +/** + * Tests the NodeRepository class used for talking to the node repository. It uses a mock from the node repository + * which already contains some data. + * + * @author dybdahl + */ public class NodeRepositoryImplTest { private JDisc container; private int port; + private final Set<HostName> configServerHosts = Sets.newHashSet(new HostName("127.0.0.1")); + private int findRandomOpenPort() throws IOException { try (ServerSocket socket = new ServerSocket(0)) { @@ -42,12 +53,13 @@ public class NodeRepositoryImplTest { @Before public void startContainer() throws Exception { port = findRandomOpenPort(); + System.err.println("PORT IS " + port); container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable); } private void waitForJdiscContainerToServe() throws InterruptedException { Instant start = Instant.now(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("foobar", "127.0.0.1", port); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(Sets.newHashSet(new HostName("127.0.0.1")), port, "foobar"); while (Instant.now().minusSeconds(120).isBefore(start)) { try { nodeRepositoryApi.getContainersToRun(); @@ -66,11 +78,10 @@ public class NodeRepositoryImplTest { } } - @Test public void testGetContainersToRunAPi() throws IOException, InterruptedException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("dockerhost4", "127.0.0.1", port); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerHosts, port, "dockerhost4"); final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun(); assertThat(containersToRun.size(), is(1)); final ContainerNodeSpec nodeSpec = containersToRun.get(0); @@ -88,9 +99,9 @@ public class NodeRepositoryImplTest { @Test public void testGetContainers() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("dockerhost4", "127.0.0.1", port); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(configServerHosts, port, "dockerhost4"); HostName hostname = new HostName("host4.yahoo.com"); - Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainer(hostname); + Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname); assertThat(nodeSpec.isPresent(), is(true)); assertThat(nodeSpec.get().hostname, is(hostname)); assertThat(nodeSpec.get().containerName, is(new ContainerName("host4"))); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java index f5c5601d661..4efe27743e3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java @@ -5,10 +5,22 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.jaxrs.client.JaxRsStrategy; import com.yahoo.vespa.jaxrs.client.LocalPassThroughJaxRsStrategy; import com.yahoo.vespa.orchestrator.restapi.HostApi; +import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi; +import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; +import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; -import static org.mockito.Matchers.anyString; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -16,16 +28,29 @@ import static org.mockito.Mockito.when; /** * @author bakksjo + * @author dybis */ public class OrchestratorImplTest { - @Test - public void redundantResumesAreFilteredOut() throws Exception { - final HostApi hostApi = mock(HostApi.class); + private HostApi hostApi; + private OrchestratorImpl orchestrator; + private HostSuspensionApi hostSuspensionApi; + final String hostNameString = "host"; + final HostName hostName = new HostName(hostNameString); + final List<String> hosts = new ArrayList<>(); + + @Before + public void before() { + hostApi = mock(HostApi.class); final JaxRsStrategy<HostApi> hostApiClient = new LocalPassThroughJaxRsStrategy<>(hostApi); - final OrchestratorImpl orchestrator = new OrchestratorImpl(hostApiClient); - final String hostNameString = "host"; - final HostName hostName = new HostName(hostNameString); + hostSuspensionApi = mock(HostSuspensionApi.class); + final JaxRsStrategy<HostSuspensionApi> hostSuspendClient = new LocalPassThroughJaxRsStrategy<>(hostSuspensionApi); + + orchestrator = new OrchestratorImpl(hostApiClient, hostSuspendClient); + } + + @Test + public void testSingleOperations() throws Exception { // Make resume and suspend always succeed. when(hostApi.resume(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null)); when(hostApi.suspend(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null)); @@ -33,18 +58,43 @@ public class OrchestratorImplTest { orchestrator.resume(hostName); verify(hostApi, times(1)).resume(hostNameString); - // A subsequent resume does not cause a network trip. - orchestrator.resume(hostName); - verify(hostApi, times(1)).resume(anyString()); - orchestrator.suspend(hostName); verify(hostApi, times(1)).suspend(hostNameString); - orchestrator.resume(hostName); - verify(hostApi, times(2)).resume(hostNameString); + hosts.add(hostNameString); + orchestrator.resume(hosts); + } - // A subsequent resume does not cause a network trip. - orchestrator.resume(hostName); - verify(hostApi, times(2)).resume(anyString()); + @Test + public void testListResumeOk() { + when(hostApi.resume(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null)); + hosts.add(hostNameString); + final Optional<String> resume = orchestrator.resume(hosts); + assertFalse(resume.isPresent()); + verify(hostApi, times(1)).resume(hostNameString); + } + + @Test + public void testListResumeFailed() { + when(hostApi.resume(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, new HostStateChangeDenialReason("", "", ""))); + hosts.add(hostNameString); + final Optional<String> resume = orchestrator.resume(hosts); + assertTrue(resume.isPresent()); + assertThat(resume.get(), is("Could not resume host")); + verify(hostApi, times(1)).resume(hostNameString); + } + + @Test + public void testListSuspendOk() throws Exception { + hosts.add(hostNameString); + when(hostSuspensionApi.suspendAll(Mockito.any())).thenReturn(new BatchOperationResult(null)); + assertThat(orchestrator.suspend("parent", hosts), is(Optional.empty())); + } + + @Test + public void testListSuspendFailed() throws Exception { + hosts.add(hostNameString); + when(hostSuspensionApi.suspendAll(Mockito.any())).thenReturn(new BatchOperationResult("no no")); + assertThat(orchestrator.suspend("parent", hosts), is(Optional.of("no no"))); } } diff --git a/parent/pom.xml b/parent/pom.xml index 11e1c96cd90..9d9586d4a7c 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -9,26 +9,35 @@ <version>6-SNAPSHOT</version> <name>parent</name> <description>Parent artifact for all Vespa maven projects.</description> - <pluginRepositories> - <pluginRepository> - <id>scala-tools.org</id> - <name>Scala-tools Maven2 Repository</name> - <url>http://scala-tools.org/repo-releases</url> - <snapshots> - <enabled>false</enabled> - </snapshots> - </pluginRepository> - </pluginRepositories> - <repositories> + <url>http://yahoo.github.io/vespa</url> + + <licenses> + <license> + <name>The Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + + <developers> + <developer> + <name>Yahoo Inc.</name> + <url>https://github.com/yahoo</url> + </developer> + </developers> + + <distributionManagement> <repository> - <id>scala-tools.org</id> - <name>Scala-tools Maven2 Repository</name> - <url>http://scala-tools.org/repo-releases</url> - <snapshots> - <enabled>false</enabled> - </snapshots> + <id>bintray-vespa-repo</id> + <url>https://api.bintray.com/maven/yahoo/maven/vespa;publish=1</url> </repository> - </repositories> + </distributionManagement> + + <scm> + <connection>scm:git:git@github.com:yahoo/vespa.git</connection> + <developerConnection>scm:git:git@github.com:yahoo/vespa.git</developerConnection> + <url>git@github.com:yahoo/vespa.git</url> + </scm> + <build> <finalName>${project.artifactId}</finalName> <extensions> diff --git a/searchlib/src/vespa/searchlib/CMakeLists.txt b/searchlib/src/vespa/searchlib/CMakeLists.txt index f0351687918..0f0420e5b61 100644 --- a/searchlib/src/vespa/searchlib/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/CMakeLists.txt @@ -2,31 +2,35 @@ vespa_add_library(searchlib SOURCES $<TARGET_OBJECTS:searchlib_aggregation> - $<TARGET_OBJECTS:searchlib_grouping> $<TARGET_OBJECTS:searchlib_attribute> + $<TARGET_OBJECTS:searchlib_bitcompression> $<TARGET_OBJECTS:searchlib_btree> $<TARGET_OBJECTS:searchlib_common> + $<TARGET_OBJECTS:searchlib_diskindex> $<TARGET_OBJECTS:searchlib_docstore> $<TARGET_OBJECTS:searchlib_engine> $<TARGET_OBJECTS:searchlib_expression> + $<TARGET_OBJECTS:searchlib_features> + $<TARGET_OBJECTS:searchlib_features_fieldmatch> + $<TARGET_OBJECTS:searchlib_features_rankingexpression> $<TARGET_OBJECTS:searchlib_fef> $<TARGET_OBJECTS:searchlib_fef_test> $<TARGET_OBJECTS:searchlib_fef_test_plugin> + $<TARGET_OBJECTS:searchlib_grouping> + $<TARGET_OBJECTS:searchlib_memoryindex> $<TARGET_OBJECTS:searchlib_parsequery> $<TARGET_OBJECTS:searchlib_predicate> + $<TARGET_OBJECTS:searchlib_query> + $<TARGET_OBJECTS:searchlib_query_tree> + $<TARGET_OBJECTS:searchlib_queryeval> + $<TARGET_OBJECTS:searchlib_queryeval_wand> $<TARGET_OBJECTS:searchlib_sconfig> - $<TARGET_OBJECTS:searchlib_searchlib_bitcompression> - $<TARGET_OBJECTS:searchlib_searchlib_diskindex> $<TARGET_OBJECTS:searchlib_searchlib_index> - $<TARGET_OBJECTS:searchlib_searchlib_memoryindex> - $<TARGET_OBJECTS:searchlib_translog> + $<TARGET_OBJECTS:searchlib_transactionlog> $<TARGET_OBJECTS:searchlib_util> + INSTALL lib64 DEPENDS - searchlib_features - searchlib_query - searchlib_queryeval - searchlib_queryeval_test staging_vespalib icuuc atomic diff --git a/searchlib/src/vespa/searchlib/bitcompression/CMakeLists.txt b/searchlib/src/vespa/searchlib/bitcompression/CMakeLists.txt index 51d299bacfa..d639bed05bf 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/bitcompression/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_searchlib_bitcompression OBJECT +vespa_add_library(searchlib_bitcompression OBJECT SOURCES compression.cpp countcompression.cpp diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt index 1cde63458ec..d9b7237c065 100644 --- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_searchlib_diskindex OBJECT +vespa_add_library(searchlib_diskindex OBJECT SOURCES bitvectordictionary.cpp bitvectorfile.cpp diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index ec21aa87fae..576e1003f50 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_features +vespa_add_library(searchlib_features OBJECT SOURCES agefeature.cpp array_parser.cpp @@ -57,8 +57,5 @@ vespa_add_library(searchlib_features utils.cpp valuefeature.cpp weighted_set_parser.cpp - $<TARGET_OBJECTS:searchlib_fieldmatch> - $<TARGET_OBJECTS:searchlib_rankingexpression> - INSTALL lib64 DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt index 2bbdf179763..965bbcf680e 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_fieldmatch OBJECT +vespa_add_library(searchlib_features_fieldmatch OBJECT SOURCES computer.cpp metrics.cpp diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt index 2853a06c49e..7c4ba6a3b9c 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/rankingexpression/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_rankingexpression OBJECT +vespa_add_library(searchlib_features_rankingexpression OBJECT SOURCES feature_name_extractor.cpp DEPENDS diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt index b9e5bf5a4ea..ba238d0eaeb 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_searchlib_memoryindex OBJECT +vespa_add_library(searchlib_memoryindex OBJECT SOURCES compact_document_words_store.cpp dictionary.cpp diff --git a/searchlib/src/vespa/searchlib/query/CMakeLists.txt b/searchlib/src/vespa/searchlib/query/CMakeLists.txt index 50aca60fc1c..d255732835e 100644 --- a/searchlib/src/vespa/searchlib/query/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/query/CMakeLists.txt @@ -1,12 +1,10 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_query +vespa_add_library(searchlib_query OBJECT SOURCES queryterm.cpp querynode.cpp base.cpp query.cpp querynoderesultbase.cpp - $<TARGET_OBJECTS:searchlib_tree> - INSTALL lib64 DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/query/tree/CMakeLists.txt b/searchlib/src/vespa/searchlib/query/tree/CMakeLists.txt index 3f7f5bdb3af..ef1af0cafcb 100644 --- a/searchlib/src/vespa/searchlib/query/tree/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/query/tree/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_tree OBJECT +vespa_add_library(searchlib_query_tree OBJECT SOURCES intermediate.cpp intermediatenodes.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index 56c77ed46cf..8e6894cc593 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_queryeval +vespa_add_library(searchlib_queryeval OBJECT SOURCES andnotsearch.cpp andsearch.cpp @@ -48,7 +48,5 @@ vespa_add_library(searchlib_queryeval unpackinfo.cpp weighted_set_term_blueprint.cpp weighted_set_term_search.cpp - $<TARGET_OBJECTS:searchlib_queryeval_wand> - INSTALL lib64 DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/CMakeLists.txt index 6b23f41a34a..1e0bcb67da6 100644 --- a/searchlib/src/vespa/searchlib/test/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_library(searchlib_test statestring.cpp initrange.cpp document_weight_attribute_helper.cpp - $<TARGET_OBJECTS:searchlib_fakedata> + $<TARGET_OBJECTS:searchlib_test_fakedata> $<TARGET_OBJECTS:searchlib_searchlib_test_diskindex> DEPENDS searchlib_searchlib_test_memoryindex diff --git a/searchlib/src/vespa/searchlib/test/fakedata/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/fakedata/CMakeLists.txt index b01ad63e02f..eb6bc335957 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/fakedata/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_fakedata OBJECT +vespa_add_library(searchlib_test_fakedata OBJECT SOURCES fakeword.cpp fakewordset.cpp diff --git a/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt b/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt index ea6b238e8b3..15fcda6730f 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/transactionlog/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchlib_translog OBJECT +vespa_add_library(searchlib_transactionlog OBJECT SOURCES common.cpp domain.cpp diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index 75e3537506d..55eef2fd8d8 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -35,7 +35,6 @@ vespa_install_script(src/vespa-start-configserver.sh vespa-start-configserver bi vespa_install_script(src/vespa-start-services.sh vespa-start-services bin) vespa_install_script(src/vespa-stop-configserver.sh vespa-stop-configserver bin) vespa_install_script(src/vespa-stop-services.sh vespa-stop-services bin) -vespa_install_script(src/vespa-core-dumper.sh vespa-core-dumper bin) configure_file(src/vespa.service.in src/vespa.service @ONLY) configure_file(src/vespa-configserver.service.in src/vespa-configserver.service @ONLY) diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index bdda265b37f..86eb96093bc 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -110,14 +110,3 @@ chown -hR ${VESPA_USER} var/db/vespa # Delete temporary files created by storage when running. rm -f /home/y/tmp/hostinfo.*.*.report - -# Add $VESPA_HOME/bin to default path -perl -pi -e 'm=^pathmunge /usr/X11R6/bin after= and s=^=pathmunge /home/y/bin after; =' /etc/profile - -#Enable core files by default -perl -pi -e 's/^# No core files by default/# Vespa: Enable core files by default/' /etc/profile -perl -pi -e 's/^ulimit -S -c 0/ulimit -S -c unlimited/' /etc/profile - -# Don't fail script if this command fails. -# * sysctl will always return error on openvz jails -sysctl kernel.core_pattern="|${VESPA_HOME}bin/vespa-core-dumper lz4 ${VESPA_HOME}var/crash/%e.core.%p.lz4" || true diff --git a/vespabase/src/vespa-core-dumper.sh b/vespabase/src/vespa-core-dumper.sh deleted file mode 100755 index 5f9baadf4fc..00000000000 --- a/vespabase/src/vespa-core-dumper.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -log_message () { - echo "warning $*" | logger -t vespa-core-dumper -} - -compressor=$1 -corefile=$2 -option=$3 - -log_message "Starting $compressor > $corefile" - - -if [ -f "$corefile" ] -then - if [ "$option" != "overwrite" ] - then - log_message "$corefile is read-only. Core is not dumped." - exit 1 - else - log_message "Overwriting $corefile" - fi -fi - -$compressor > $corefile -chmod 444 $corefile -log_message "Finished $compressor > $corefile" diff --git a/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java index 34a7a9b9c6e..5a1f6c249f2 100644 --- a/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java +++ b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java @@ -20,8 +20,6 @@ import java.util.stream.Collectors; */ public class LinuxInetAddress { - private static Logger log = Logger.getLogger(LinuxInetAddress.class.getName()); - /** * Returns an InetAddress representing the address of the localhost. * A non-loopback address is preferred if available. diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp index 0eb08a80f41..5deab82b6fc 100644 --- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp @@ -116,4 +116,45 @@ TEST_F("requireThatNewTasksAreDroppedAfterShutdown", MyState()) { TEST_DO(f1.open().shutdown().execute(5).sync().check(5, 0, 0, 0)); } + +struct WaitTask : public Executor::Task { + Gate &gate; + WaitTask(Gate &g) : gate(g) {} + virtual void run() { gate.await(); } +}; + +struct WaitState { + ThreadStackExecutor executor; + std::vector<Gate> block_task; + std::vector<Gate> wait_done; + WaitState(size_t num_threads) + : executor(num_threads / 2, 128000), block_task(num_threads - 2), wait_done(num_threads - 1) + { + for (auto &gate: block_task) { + auto result = executor.execute(std::make_unique<WaitTask>(gate)); + ASSERT_TRUE(result.get() == nullptr); + } + } + void wait(size_t count) { + executor.wait_for_task_count(count); + wait_done[count].countDown(); + } +}; + +TEST_MT_F("require that threads can wait for a specific task count", 7, WaitState(num_threads)) { + if (thread_id == 0) { + for (size_t next_done = (num_threads - 2); next_done-- > 0;) { + if (next_done < f1.block_task.size()) { + f1.block_task[f1.block_task.size() - 1 - next_done].countDown(); + } + EXPECT_TRUE(f1.wait_done[next_done].await(25000)); + for (size_t i = 0; i < next_done; ++i) { + EXPECT_TRUE(!f1.wait_done[i].await(20)); + } + } + } else { + f1.wait(thread_id - 1); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/tutorial/CMakeLists.txt b/vespalib/src/tests/tutorial/CMakeLists.txt index 40f12c42edb..120c848b2ae 100644 --- a/vespalib/src/tests/tutorial/CMakeLists.txt +++ b/vespalib/src/tests/tutorial/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_executable(vespalib_make_tutorial_app TEST DEPENDS vespalib ) -vespa_add_test(NAME vespalib_make_tutorial_app COMMAND vespalib_make_tutorial_app) +vespa_add_test(NAME vespalib_make_tutorial_app COMMAND sh ./compare-tutorials.sh) vespa_add_executable(vespalib_xml_escape_app SOURCES xml_escape.cpp diff --git a/vespalib/src/tests/tutorial/compare-tutorials.sh b/vespalib/src/tests/tutorial/compare-tutorials.sh new file mode 100755 index 00000000000..e3b7ed8ccf5 --- /dev/null +++ b/vespalib/src/tests/tutorial/compare-tutorials.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +./vespalib_make_tutorial_app > tutorial_out.html && diff -u tutorial.html tutorial_out.html diff --git a/vespalib/src/tests/tutorial/make_example.sh b/vespalib/src/tests/tutorial/make_example.sh index 9b015c93e2b..b5919254bc9 100755 --- a/vespalib/src/tests/tutorial/make_example.sh +++ b/vespalib/src/tests/tutorial/make_example.sh @@ -13,7 +13,7 @@ echo "<pre class=\"prettyprint linenums\">" (cd $dirname && cat $filename) | ./vespalib_xml_escape_app echo "</pre>" echo "<pre class=\"output\">" -(cd $dirname && make all > /dev/null 2>&1) -(cd $dirname && make test 2>&1) | ./vespalib_xml_escape_app +DIRNAME=`(cd $dirname && /bin/pwd)` +(cd $dirname && ./vespalib_${filename%.cpp}_app 2>&1) | perl -pe "s{$DIRNAME/}{}g" | ./vespalib_xml_escape_app echo "</pre>" echo "</div>" diff --git a/vespalib/src/tests/tutorial/tutorial.html b/vespalib/src/tests/tutorial/tutorial.html index 744dd9541e2..8b6706566f0 100644 --- a/vespalib/src/tests/tutorial/tutorial.html +++ b/vespalib/src/tests/tutorial/tutorial.html @@ -190,7 +190,6 @@ checks_test.cpp: info: test summary --- 1 test(s) passed --- 1 test(s) failed checks_test.cpp: info: imported 11 passed check(s) from 1 thread(s) checks_test.cpp: info: summary --- 11 check(s) passed --- 1 check(s) failed checks_test.cpp: info: CONCLUSION: FAIL -make: *** [test] Error 1 </pre> </div> diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index bee1d9c17ec..780f0ed5dd0 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -6,6 +6,47 @@ namespace vespalib { void +ThreadStackExecutorBase::BlockedThread::wait() const +{ + MonitorGuard guard(monitor); + while (blocked) { + guard.wait(); + } +} + +void +ThreadStackExecutorBase::BlockedThread::unblock() +{ + MonitorGuard guard(monitor); + blocked = false; + guard.signal(); +} + +//----------------------------------------------------------------------------- + +void +ThreadStackExecutorBase::block_thread(const LockGuard &, BlockedThread &blocked_thread) +{ + auto pos = _blocked.begin(); + while ((pos != _blocked.end()) && + ((*pos)->wait_task_count < blocked_thread.wait_task_count)) + { + ++pos; + } + _blocked.insert(pos, &blocked_thread); +} + +void +ThreadStackExecutorBase::unblock_threads(const MonitorGuard &) +{ + while (!_blocked.empty() && (_taskCount <= _blocked.back()->wait_task_count)) { + BlockedThread &blocked_thread = *(_blocked.back()); + _blocked.pop_back(); + blocked_thread.unblock(); + } +} + +void ThreadStackExecutorBase::assignTask(const TaggedTask &task, Worker &worker) { MonitorGuard monitor(worker.monitor); @@ -26,6 +67,7 @@ ThreadStackExecutorBase::obtainTask(Worker &worker) _barrier.completeEvent(worker.task.token); worker.task.task = 0; } + unblock_threads(monitor); if (!_tasks.empty()) { worker.task = _tasks.front(); _tasks.pop(); @@ -168,6 +210,19 @@ ThreadStackExecutorBase::sync() } void +ThreadStackExecutorBase::wait_for_task_count(uint32_t task_count) +{ + LockGuard lock(_monitor); + if (_taskCount <= task_count) { + return; + } + BlockedThread self(task_count); + block_thread(lock, self); + lock.unlock(); // <- UNLOCK + self.wait(); +} + +void ThreadStackExecutorBase::cleanup() { shutdown().sync(); @@ -179,6 +234,7 @@ ThreadStackExecutorBase::~ThreadStackExecutorBase() { assert(_pool.isClosed()); assert(_taskCount == 0); + assert(_blocked.empty()); } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index f66cd74dae1..215a8377fc6 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -52,17 +52,31 @@ private: void completeBarrier() { gate.countDown(); } }; + struct BlockedThread { + const uint32_t wait_task_count; + Monitor monitor; + bool blocked; + BlockedThread(uint32_t wait_task_count_in) + : wait_task_count(wait_task_count_in), monitor(), blocked(true) {} + void wait() const; + void unblock(); + }; + FastOS_ThreadPool _pool; Monitor _monitor; Stats _stats; Gate _executorCompletion; ArrayQueue<TaggedTask> _tasks; ArrayQueue<Worker*> _workers; + std::vector<BlockedThread*> _blocked; EventBarrier<BarrierCompletion> _barrier; uint32_t _taskCount; uint32_t _taskLimit; bool _closed; + void block_thread(const LockGuard &, BlockedThread &blocked_thread); + void unblock_threads(const MonitorGuard &); + /** * Assign the given task to the given idle worker. This will wake * up a worker thread that is blocked in the obtainTask function. @@ -153,6 +167,14 @@ public: virtual ThreadStackExecutorBase &sync(); /** + * Block the calling thread until the current task count is equal + * to or lower than the given value. + * + * @param task_count target value to wait for + **/ + void wait_for_task_count(uint32_t task_count); + + /** * Shut down this executor. This will make this executor reject * all new tasks. * |