diff options
10 files changed, 69 insertions, 43 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java index f3b529d8c05..704217843fb 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.applicationmodel; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; +import java.util.List; +import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -24,6 +26,13 @@ public enum InfrastructureApplication { private final ApplicationId id; private final NodeType nodeType; + /** Returns all applications that MAY be encountered in hosted Vespa, e.g. not DEV_HOST. */ + public static List<InfrastructureApplication> inHosted() { + return Stream.of(values()) + .filter(application -> application != DEV_HOST) + .collect(Collectors.toList()); + } + public static InfrastructureApplication withNodeType(NodeType nodeType) { return Stream.of(values()) .filter(application -> nodeType == application.nodeType) diff --git a/config-model/src/test/cfg/application/app_complicated_deployment_spec/services.xml b/config-model/src/test/cfg/application/app_complicated_deployment_spec/services.xml index 02d41eac283..d873129b2c7 100644 --- a/config-model/src/test/cfg/application/app_complicated_deployment_spec/services.xml +++ b/config-model/src/test/cfg/application/app_complicated_deployment_spec/services.xml @@ -2,33 +2,6 @@ <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0"> - <service version="1.0" name="myservice" command="mycmd1.sh"> - <config name="a.myconfig"> - <mysetting>bar</mysetting> - </config> - <node hostalias="node1"> - <config name="a.myconfig"> - <mysetting>baz</mysetting> - </config> - </node> - <node hostalias="node2"/> - <node hostalias="node3"/> - <node hostalias="node3"/> - </service> - - <service version="1.0" name="myotherservice" command="/home/vespa/bin/mycmd2.sh --ytest $FOO_BAR"> - <config name="a.myconfig"> - <mysetting>bar2</mysetting> - </config> - <node hostalias="node3"> - <config name="a.myconfig"> - <mysetting>baz2</mysetting> - </config> - </node> - <node hostalias="node4"/> - <node hostalias="node5"/> - </service> - <admin version="2.0"> <adminserver hostalias="node1"/> <slobroks> diff --git a/controller-server/pom.xml b/controller-server/pom.xml index e9fadee58c7..fa4a0dc06d6 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -70,6 +70,13 @@ <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>application-model</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespa-athenz</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 2819e382017..613422b2749 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -3,11 +3,11 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; +import com.yahoo.vespa.applicationmodel.InfrastructureApplication; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; @@ -26,33 +26,31 @@ import java.util.Optional; */ public enum SystemApplication { - controllerHost("controller-host", NodeType.controllerhost), - configServerHost("configserver-host", NodeType.confighost), - configServer("zone-config-servers", NodeType.config), - proxyHost("proxy-host", NodeType.proxyhost), - proxy( "routing", NodeType.proxy, proxyHost, configServer), - tenantHost("tenant-host", NodeType.host); + controllerHost(InfrastructureApplication.CONTROLLER_HOST), + configServerHost(InfrastructureApplication.CONFIG_SERVER_HOST), + configServer(InfrastructureApplication.CONFIG_SERVER), + proxyHost(InfrastructureApplication.PROXY_HOST), + proxy(InfrastructureApplication.PROXY, proxyHost, configServer), + tenantHost(InfrastructureApplication.TENANT_HOST); /** The tenant owning all system applications */ public static final TenantName TENANT = TenantName.from(Constants.TENANT_NAME); - private final ApplicationId id; - private final NodeType nodeType; + private final InfrastructureApplication application; private final List<SystemApplication> dependencies; - SystemApplication(String application, NodeType nodeType, SystemApplication... dependencies) { - this.id = ApplicationId.from(Constants.TENANT_NAME, application, InstanceName.defaultName().value()); - this.nodeType = nodeType; + SystemApplication(InfrastructureApplication application, SystemApplication... dependencies) { + this.application = application; this.dependencies = List.of(dependencies); } public ApplicationId id() { - return id; + return application.id(); } /** The node type that is implicitly allocated to this */ public NodeType nodeType() { - return nodeType; + return application.nodeType(); } /** Returns the system applications that should upgrade before this */ @@ -75,7 +73,7 @@ public enum SystemApplication { /** Returns whether this should receive OS upgrades */ public boolean shouldUpgradeOs() { - return nodeType.isHost(); + return nodeType().isHost(); } /** Returns whether this has an endpoint */ @@ -106,7 +104,7 @@ public enum SystemApplication { @Override public String toString() { - return Text.format("system application %s of type %s", id, nodeType); + return Text.format("system application %s of type %s", id(), nodeType()); } private static class Constants { diff --git a/dist/vespa.spec b/dist/vespa.spec index 8f380383d00..31eafe51ae2 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -874,6 +874,7 @@ fi %dir %{_prefix}/lib/jars %{_prefix}/lib/jars/asm-*.jar %{_prefix}/lib/jars/aopalliance-repackaged-*.jar +%{_prefix}/lib/jars/application-model-jar-with-dependencies.jar %{_prefix}/lib/jars/bcpkix-jdk15on-*.jar %{_prefix}/lib/jars/bcprov-jdk15on-*.jar %{_prefix}/lib/jars/config-bundle-jar-with-dependencies.jar diff --git a/node-admin/CMakeLists.txt b/node-admin/CMakeLists.txt index 3e0cec54132..8ed90d6b8f1 100644 --- a/node-admin/CMakeLists.txt +++ b/node-admin/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install(DIRECTORY DESTINATION logs/vespa/node-admin) install(FILES target/node-admin-jar-with-dependencies.jar DESTINATION conf/node-admin-app/components) +install_symlink(lib/jars/application-model-jar-with-dependencies.jar conf/node-admin-app/components/application-model-jar-with-dependencies.jar) install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/node-admin-app/components/flags-jar-with-dependencies.jar) install(FILES src/main/application/services.xml DESTINATION conf/node-admin-app) install(PROGRAMS src/main/sh/node-admin.sh DESTINATION libexec/vespa) diff --git a/searchlib/src/tests/transactionlog/chunks_test.cpp b/searchlib/src/tests/transactionlog/chunks_test.cpp index 8d6620c9f8b..ca03a47634d 100644 --- a/searchlib/src/tests/transactionlog/chunks_test.cpp +++ b/searchlib/src/tests/transactionlog/chunks_test.cpp @@ -115,4 +115,25 @@ TEST("test multi element commitchunk") { } EXPECT_EQUAL(0u, counter); } + +TEST("shrinkToFit if difference is larger than 8x") { + Packet p(16000); + p.add(Packet::Entry(1, 3, ConstBufferRef(TEXT, strlen(TEXT)))); + EXPECT_EQUAL(150u, p.sizeBytes()); + EXPECT_EQUAL(16384u, p.getHandle().capacity()); + p.shrinkToFit(); + EXPECT_EQUAL(150u, p.sizeBytes()); + EXPECT_EQUAL(150u, p.getHandle().capacity()); +} + +TEST("not shrinkToFit if difference is less than 8x") { + Packet p(1000); + p.add(Packet::Entry(1, 3, ConstBufferRef(TEXT, strlen(TEXT)))); + EXPECT_EQUAL(150u, p.sizeBytes()); + EXPECT_EQUAL(1024u, p.getHandle().capacity()); + p.shrinkToFit(); + EXPECT_EQUAL(150u, p.sizeBytes()); + EXPECT_EQUAL(1024u, p.getHandle().capacity()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.cpp b/searchlib/src/vespa/searchlib/transactionlog/common.cpp index 967f8748174..4130ad0bc06 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/common.cpp @@ -116,6 +116,14 @@ Packet::add(const Packet::Entry & e) _range.to(e.serial()); } +void +Packet::shrinkToFit() { + if (_buf.size() * 8 < _buf.capacity()) { + nbostream::Buffer shrunkToFit(_buf.data(), _buf.data() + _buf.size()); + _buf.swap(shrunkToFit); + } +} + Writer::CommitResult::CommitResult() : _callBacks() {} @@ -151,4 +159,9 @@ CommitChunk::createCommitResult() const { return Writer::CommitResult(_callBacks); } +void +CommitChunk::shrinkPayloadToFit() { + _data.shrinkToFit(); +} + } diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.h b/searchlib/src/vespa/searchlib/transactionlog/common.h index 135681037ef..87150f2cfa9 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.h +++ b/searchlib/src/vespa/searchlib/transactionlog/common.h @@ -83,6 +83,7 @@ public: bool empty() const { return _count == 0; } size_t sizeBytes() const { return _buf.size(); } void merge(const Packet & packet); + void shrinkToFit(); private: size_t _count; SerialNumRange _range; @@ -143,6 +144,7 @@ public: Writer::CommitResult createCommitResult() const; void setCommitDoneCallback(Writer::DoneCallback onDone) { _onCommitDone = std::move(onDone); } Writer::CommitPayload stealCallbacks() { return std::move(_callBacks); } + void shrinkPayloadToFit(); private: Packet _data; Writer::CommitPayload _callBacks; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index bbde39a42f6..ff35847aa77 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -396,6 +396,7 @@ void Domain::commitChunk(std::unique_ptr<CommitChunk> chunk, const UniqueLock & chunkOrderGuard) { assert(chunkOrderGuard.mutex() == &_currentChunkMonitor && chunkOrderGuard.owns_lock()); if (chunk->getPacket().empty()) return; + chunk->shrinkPayloadToFit(); std::promise<SerializedChunk> promise; std::future<SerializedChunk> future = promise.get_future(); _executor.execute(makeLambdaTask([promise=std::move(promise), chunk = std::move(chunk), |