aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2018-06-01 11:31:13 +0200
committerJon Bratseth <bratseth@oath.com>2018-06-01 11:31:13 +0200
commitcd6e2dd7ab638487c462c40293a0d8db43fdb530 (patch)
treeedf9cec58f4a0e995fd713089779ec5d0a14d134
parentf7d0fe5fdfad579e867f01acc1d41b2ec77bd785 (diff)
parent5c57852b26126d72b080fb6e0893dc3d633c28c1 (diff)
Merge with master
-rw-r--r--application/src/main/java/com/yahoo/application/container/JDisc.java7
-rw-r--r--athenz-identity-provider-service/pom.xml8
-rw-r--r--athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java28
-rw-r--r--athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java18
-rw-r--r--athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java1
-rw-r--r--athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java9
-rw-r--r--athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java7
-rwxr-xr-xconfig-lib/src/main/java/com/yahoo/config/FileReference.java5
-rw-r--r--config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java6
-rw-r--r--config-model-fat/pom.xml200
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java84
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java36
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java20
-rw-r--r--config-model/src/test/derived/map_of_struct_attribute/attributes.cfg60
-rw-r--r--config-model/src/test/derived/map_of_struct_attribute/summary.cfg11
-rw-r--r--config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg7
-rw-r--r--config-model/src/test/derived/map_of_struct_attribute/test.sd20
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java11
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java10
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java10
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java93
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java7
-rw-r--r--configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java5
-rw-r--r--configdefinitions/src/vespa/athenz-provider-service.def (renamed from athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def)0
-rw-r--r--configdefinitions/src/vespa/configserver.def5
-rw-r--r--configgen/pom.xml34
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java351
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java444
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java2
-rw-r--r--configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java170
-rw-r--r--configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala350
-rw-r--r--configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala423
-rw-r--r--configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala186
-rw-r--r--configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java116
-rw-r--r--configgen/src/test/resources/allfeatures.reference1983
-rw-r--r--configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala97
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java87
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java14
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java5
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java13
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java13
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java)11
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java6
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java10
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java12
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java33
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java3
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java11
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java9
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java81
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java21
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java25
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java28
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java9
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java23
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java9
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java2
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java20
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java12
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java14
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java2
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java16
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java7
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java6
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java8
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java3
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java32
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java2
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java334
-rw-r--r--container-search/src/main/java/com/yahoo/search/result/Hit.java28
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java67
-rw-r--r--container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java2
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java59
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java31
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java10
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java1
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java9
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json3
-rw-r--r--document/src/tests/documentupdatetestcase.cpp55
-rw-r--r--document/src/tests/fieldpathupdatetestcase.cpp8
-rw-r--r--document/src/vespa/document/serialization/vespadocumentserializer.h2
-rw-r--r--document/src/vespa/document/update/documentupdate.cpp58
-rw-r--r--document/src/vespa/document/update/documentupdate.h68
-rw-r--r--document/src/vespa/document/update/fieldpathupdate.cpp12
-rw-r--r--document/src/vespa/document/update/fieldpathupdate.h3
-rw-r--r--document/src/vespa/document/update/fieldupdate.cpp34
-rw-r--r--document/src/vespa/document/update/fieldupdate.h2
-rw-r--r--documentapi/src/tests/messages/messages50test.cpp3
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp413
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp23
-rw-r--r--fat-model-dependencies/OWNERS2
-rw-r--r--fat-model-dependencies/README4
-rw-r--r--fat-model-dependencies/pom.xml223
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java105
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java1
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java42
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java9
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java3
-rw-r--r--pom.xml1
-rw-r--r--searchcore/src/tests/proton/attribute/attribute_test.cpp179
-rw-r--r--searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp33
-rw-r--r--searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp53
-rw-r--r--searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp7
-rw-r--r--searchcore/src/tests/proton/matching/matching_test.cpp73
-rw-r--r--searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp142
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp169
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h31
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp17
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h9
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h23
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h20
-rw-r--r--searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp18
-rw-r--r--searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h172
-rw-r--r--searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h3
-rw-r--r--searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp36
-rw-r--r--searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h10
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt1
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp13
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h1
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp96
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h24
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp43
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h7
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp25
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h49
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp56
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h20
-rw-r--r--searchlib/src/vespa/searchlib/attribute/attributeiterators.h2
-rw-r--r--searchlib/src/vespa/searchlib/attribute/attributevector.cpp78
-rw-r--r--searchlib/src/vespa/searchlib/attribute/attributevector.h28
-rw-r--r--searchlib/src/vespa/searchlib/attribute/attrvector.hpp2
-rw-r--r--searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp24
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp8
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/searchiterator.h3
-rw-r--r--standalone-container/pom.xml39
-rw-r--r--standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java569
-rw-r--r--standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java34
-rw-r--r--standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java97
-rw-r--r--standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java304
-rw-r--r--standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java131
-rw-r--r--standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala206
-rw-r--r--standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala27
-rw-r--r--standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala26
-rw-r--r--standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala23
-rw-r--r--standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala75
-rw-r--r--standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala231
-rw-r--r--standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala81
-rw-r--r--standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java67
-rw-r--r--standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java61
-rw-r--r--standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java5
-rw-r--r--standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java74
-rw-r--r--standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java52
-rw-r--r--standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala59
-rw-r--r--standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala64
-rw-r--r--standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala85
-rw-r--r--standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala41
-rw-r--r--storage/src/tests/storageserver/communicationmanagertest.cpp45
-rw-r--r--storage/src/vespa/storage/distributor/pendingclusterstate.cpp1
-rw-r--r--storage/src/vespa/storage/storageserver/communicationmanager.cpp14
-rw-r--r--storage/src/vespa/storage/storageserver/storagenode.cpp10
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp227
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h53
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp8
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h2
-rw-r--r--storageserver/src/tests/testhelper.cpp6
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java22
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java2
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityType.java25
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java39
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java46
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java6
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java2
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java41
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java26
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java3
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java13
-rw-r--r--vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java13
-rw-r--r--vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java10
-rw-r--r--vespalib/src/vespa/vespalib/objects/nbostream.cpp2
-rw-r--r--vespalib/src/vespa/vespalib/util/alloc.cpp59
-rw-r--r--vespalib/src/vespa/vespalib/util/alloc.h6
192 files changed, 7280 insertions, 4255 deletions
diff --git a/application/src/main/java/com/yahoo/application/container/JDisc.java b/application/src/main/java/com/yahoo/application/container/JDisc.java
index ed0c29a3917..5554ae6a159 100644
--- a/application/src/main/java/com/yahoo/application/container/JDisc.java
+++ b/application/src/main/java/com/yahoo/application/container/JDisc.java
@@ -15,7 +15,6 @@ import com.yahoo.component.provider.ComponentRegistry;
import com.yahoo.config.model.ConfigModelRepo;
import com.yahoo.container.Container;
import com.yahoo.container.standalone.StandaloneContainerApplication;
-import com.yahoo.container.standalone.StandaloneContainerApplication$;
import com.yahoo.docproc.jdisc.DocumentProcessingHandler;
import com.yahoo.io.IOUtils;
import com.yahoo.jdisc.handler.RequestHandler;
@@ -58,10 +57,10 @@ public final class JDisc implements AutoCloseable {
return new AbstractModule() {
@Override
protected void configure() {
- bind(Path.class).annotatedWith(StandaloneContainerApplication.applicationPathName()).toInstance(path);
- bind(ConfigModelRepo.class).annotatedWith(StandaloneContainerApplication.configModelRepoName()).toInstance(configModelRepo);
+ bind(Path.class).annotatedWith(StandaloneContainerApplication.APPLICATION_PATH_NAME).toInstance(path);
+ bind(ConfigModelRepo.class).annotatedWith(StandaloneContainerApplication.CONFIG_MODEL_REPO_NAME).toInstance(configModelRepo);
bind(Boolean.class).annotatedWith( // below is an ugly hack to access fields from a scala object.
- Names.named(StandaloneContainerApplication$.MODULE$.disableNetworkingAnnotation())).toInstance(
+ Names.named(StandaloneContainerApplication.DISABLE_NETWORKING_ANNOTATION)).toInstance(
networking == Networking.disable);
}
};
diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml
index 86d4defa861..982cb89f2bf 100644
--- a/athenz-identity-provider-service/pom.xml
+++ b/athenz-identity-provider-service/pom.xml
@@ -131,6 +131,14 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <arg>-Xlint:all</arg>
+ <arg>-Xlint:-deprecation</arg>
+ <arg>-Xlint:-serial</arg>
+ <arg>-Werror</arg>
+ </compilerArgs>
+ </configuration>
</plugin>
</plugins>
</build>
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java
index 728406c297f..59126fd023f 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java
@@ -7,6 +7,7 @@ import com.yahoo.net.HostName;
import com.yahoo.vespa.athenz.api.AthenzService;
import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument;
+import com.yahoo.vespa.athenz.identityprovider.api.IdentityType;
import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId;
import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider;
@@ -27,7 +28,10 @@ import java.util.Objects;
import java.util.Set;
/**
+ * Generates a signed identity document for a given hostname and type
+ *
* @author mortent
+ * @author bjorncs
*/
public class IdentityDocumentGenerator {
@@ -47,10 +51,10 @@ public class IdentityDocumentGenerator {
this.keyProvider = keyProvider;
}
- public SignedIdentityDocument generateSignedIdentityDocument(String hostname) {
+ public SignedIdentityDocument generateSignedIdentityDocument(String hostname, IdentityType identityType) {
Node node = nodeRepository.getNode(hostname).orElseThrow(() -> new RuntimeException("Unable to find node " + hostname));
try {
- IdentityDocument identityDocument = generateIdDocument(node);
+ IdentityDocument identityDocument = generateIdDocument(node, identityType);
String identityDocumentString = Utils.getMapper().writeValueAsString(EntityBindingsMapper.toIdentityDocumentEntity(identityDocument));
String encodedIdentityDocument =
@@ -70,13 +74,18 @@ public class IdentityDocumentGenerator {
toZoneDnsSuffix(zone, zoneConfig.certDnsSuffix()),
new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()),
URI.create(zoneConfig.ztsUrl()),
- SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION);
+ SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION,
+ identityDocument.configServerHostname(),
+ identityDocument.instanceHostname(),
+ identityDocument.createdAt(),
+ identityDocument.ipAddresses(),
+ identityType);
} catch (Exception e) {
throw new RuntimeException("Exception generating identity document: " + e.getMessage(), e);
}
}
- private IdentityDocument generateIdDocument(Node node) {
+ private IdentityDocument generateIdDocument(Node node, IdentityType identityType) {
Allocation allocation = node.allocation().orElseThrow(() -> new RuntimeException("No allocation for node " + node.hostname()));
VespaUniqueInstanceId providerUniqueId = new VespaUniqueInstanceId(
allocation.membership().index(),
@@ -85,17 +94,10 @@ public class IdentityDocumentGenerator {
allocation.owner().application().value(),
allocation.owner().tenant().value(),
zone.region().value(),
- zone.environment().value());
+ zone.environment().value(),
+ identityType);
- // TODO: Hack to allow access from docker containers to non-ipv6 services.
- // Remove when yca-bridge is no longer needed
Set<String> ips = new HashSet<>(node.ipAddresses());
- if(node.parentHostname().isPresent()) {
- String parentHostName = node.parentHostname().get();
- nodeRepository.getNode(parentHostName)
- .map(Node::ipAddresses)
- .ifPresent(ips::addAll);
- }
return new IdentityDocument(
providerUniqueId,
HostName.getLocalhost(),
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java
index 93668006e26..219e12c7223 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java
@@ -6,6 +6,7 @@ import com.yahoo.container.jaxrs.annotation.Component;
import com.yahoo.jdisc.http.servlet.ServletRequest;
import com.yahoo.log.LogLevel;
import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
+import com.yahoo.vespa.athenz.identityprovider.api.IdentityType;
import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentApi;
import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity;
import com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodePrincipal;
@@ -18,7 +19,6 @@ import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.util.logging.Logger;
@@ -41,15 +41,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi {
this.request = request;
}
- /**
- * @deprecated Use {@link #getNodeIdentityDocument(String)} and {@link #getTenantIdentityDocument(String)} instead.
- */
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- @Deprecated
- @Override
- // TODO Make this method private when the rest api is not longer in use
- public SignedIdentityDocumentEntity getIdentityDocument(@QueryParam("hostname") String hostname) {
+ private SignedIdentityDocumentEntity getIdentityDocument(String hostname, IdentityType identityType) {
if (hostname == null) {
throw new BadRequestException("The 'hostname' query parameter is missing");
}
@@ -67,7 +59,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi {
throw new ForbiddenException();
}
try {
- return EntityBindingsMapper.toSignedIdentityDocumentEntity(identityDocumentGenerator.generateSignedIdentityDocument(hostname));
+ return EntityBindingsMapper.toSignedIdentityDocumentEntity(identityDocumentGenerator.generateSignedIdentityDocument(hostname, identityType));
} catch (Exception e) {
String message = String.format("Unable to generate identity doument for '%s': %s", hostname, e.getMessage());
log.log(LogLevel.ERROR, message, e);
@@ -80,7 +72,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi {
@Path("/node/{host}")
@Override
public SignedIdentityDocumentEntity getNodeIdentityDocument(@PathParam("host") String host) {
- return getIdentityDocument(host);
+ return getIdentityDocument(host, IdentityType.NODE);
}
@GET
@@ -88,7 +80,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi {
@Path("/tenant/{host}")
@Override
public SignedIdentityDocumentEntity getTenantIdentityDocument(@PathParam("host") String host) {
- return getIdentityDocument(host);
+ return getIdentityDocument(host, IdentityType.TENANT);
}
}
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java
index e457df37946..0201c46b253 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java
@@ -82,6 +82,7 @@ public class InstanceValidator {
}
// If/when we dont care about logging exactly whats wrong, this can be simplified
+ // TODO Use identity type to determine if this check should be performed
boolean isSameIdentityAsInServicesXml(ApplicationId applicationId, String domain, String service) {
Optional<ApplicationInfo> applicationInfo = superModelProvider.getSuperModel().getApplicationInfo(applicationId);
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java
index d7b061ca2f1..078ef1b7e39 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java
@@ -15,6 +15,7 @@ import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
+import com.yahoo.vespa.athenz.identityprovider.api.IdentityType;
import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId;
import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity;
@@ -81,7 +82,7 @@ public class IdentityDocumentGeneratorTest {
AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix, ZONE);
IdentityDocumentGenerator identityDocumentGenerator =
new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider);
- SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname);
+ SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname, IdentityType.TENANT);
// Verify attributes
assertEquals(containerHostname, signedIdentityDocument.identityDocument().instanceHostname());
@@ -92,11 +93,11 @@ public class IdentityDocumentGeneratorTest {
assertEquals(expectedZoneDnsSuffix, signedIdentityDocument.dnsSuffix());
VespaUniqueInstanceId expectedProviderUniqueId =
- new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", region, environment);
+ new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", region, environment, IdentityType.TENANT);
assertEquals(expectedProviderUniqueId, signedIdentityDocument.providerUniqueId());
- // Validate that both parent and container ips are present
- assertThat(signedIdentityDocument.identityDocument().ipAddresses(), Matchers.containsInAnyOrder("127.0.0.1", "::1"));
+ // Validate that container ips are present
+ assertThat(signedIdentityDocument.identityDocument().ipAddresses(), Matchers.containsInAnyOrder("::1"));
SignedIdentityDocumentEntity signedIdentityDocumentEntity = EntityBindingsMapper.toSignedIdentityDocumentEntity(signedIdentityDocument);
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
index 54786c86cd3..54411b424eb 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
@@ -143,7 +143,12 @@ public class InstanceValidatorTest {
"dnssuffix",
"service",
URI.create("http://localhost/zts"),
- 1));
+ 1,
+ identityDocument.configServerHostname,
+ identityDocument.instanceHostname,
+ identityDocument.createdAt,
+ identityDocument.ipAddresses,
+ null)); // TODO Remove support for legacy representation without type
} catch (Exception e) {
throw new RuntimeException(e);
}
diff --git a/config-lib/src/main/java/com/yahoo/config/FileReference.java b/config-lib/src/main/java/com/yahoo/config/FileReference.java
index 5f0bc275bad..7d455c58b30 100755
--- a/config-lib/src/main/java/com/yahoo/config/FileReference.java
+++ b/config-lib/src/main/java/com/yahoo/config/FileReference.java
@@ -7,19 +7,20 @@ import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
/**
* An immutable file reference that can only be created from classes within the same package.
* This is to prevent clients from creating arbitrary and invalid file references.
*
- * @author tonytv
+ * @author Tony Vaagenes
*/
public final class FileReference {
private final String value;
public FileReference(String value) {
- this.value = value;
+ this.value = Objects.requireNonNull(value);
}
public String value() {
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
index ac1bcfa542a..9b457f49bd2 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
@@ -23,12 +23,18 @@ public interface FileDistribution {
*/
void startDownload(String hostName, int port, Set<FileReference> fileReferences);
+ // TODO: Remove when 6.244 is oldest version in use
+ @Deprecated
static String getDefaultFileDBRoot() {
return Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution");
}
+ // TODO: Remove when 6.244 is oldest version in use
+ @Deprecated
static File getDefaultFileDBPath() {
return new File(getDefaultFileDBRoot());
}
+ File getFileReferencesDir();
+
}
diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml
index 1857cc33d64..3ef9925510c 100644
--- a/config-model-fat/pom.xml
+++ b/config-model-fat/pom.xml
@@ -14,30 +14,29 @@
<dependencies>
<dependency>
<groupId>com.yahoo.vespa</groupId>
- <artifactId>config-model-api</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>config-provisioning</artifactId>
+ <artifactId>fat-model-dependencies</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
+ <type>pom</type>
</dependency>
+
<dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>config-model</artifactId>
- <version>${project.version}</version>
+ <!-- TODO: remove, we probably don't need version 13. -->
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>13.0.1</version>
</dependency>
+
<dependency>
<groupId>com.yahoo.vespa</groupId>
- <artifactId>config-lib</artifactId>
+ <artifactId>config-model-api</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
- <artifactId>provided-dependencies</artifactId>
+ <artifactId>config-provisioning</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>com.google.inject</groupId>
@@ -45,6 +44,8 @@
</exclusion>
</exclusions>
</dependency>
+
+ <!-- TODO: remove all test deps, should not be needed -->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
@@ -60,35 +61,6 @@
<artifactId>guava-testlib</artifactId>
<version>17.0</version>
<scope>test</scope>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>13.0.1</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>configdefinitions</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>config-application-package</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>configgen</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>config-bundle</artifactId>
- <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.yahoo.vespa</groupId>
@@ -102,109 +74,9 @@
</dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
- <artifactId>simplemetrics</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>metrics</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>container-disc</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>vespajlib</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>yolean</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
<artifactId>testutil</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>documentapi</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>vdslib</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>messagebus</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>document</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>container-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>linguistics</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>vespalog</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>statistics</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>messagebus-disc</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>container-messagebus</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>searchlib</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>processing</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>chain</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>docproc</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>container-search</artifactId>
- <version>${project.version}</version>
<exclusions>
<exclusion>
<!-- OPTIMIZATION: very large (44 MB) and only used for query sorting -->
@@ -213,50 +85,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>container-search-and-docproc</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>logd</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>searchcore</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>storage</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>vsm</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>indexinglanguage</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>searchsummary</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.major-version}</artifactId>
- </dependency>
- <dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>jdisc_http_service</artifactId>
- <version>${project.version}</version>
- </dependency>
</dependencies>
<build>
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index c7ca1a33ff2..fc75f7a19fc 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -5,6 +5,7 @@ import com.yahoo.config.subscription.ConfigInstanceUtil;
import com.yahoo.document.ArrayDataType;
import com.yahoo.document.DataType;
import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
import com.yahoo.document.PositionDataType;
import com.yahoo.document.StructDataType;
import com.yahoo.searchdefinition.Search;
@@ -44,39 +45,59 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
/** Derives everything from a field */
@Override
protected void derive(ImmutableSDField field, Search search) {
- boolean fieldIsArrayOfSimpleStruct = isArrayOfSimpleStruct(field);
- if (field.usesStructOrMap() &&
- !fieldIsArrayOfSimpleStruct &&
- !field.getDataType().equals(PositionDataType.INSTANCE) &&
- !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) {
- return; // Ignore struct fields for indexed search (only implemented for streaming search)
+ if (unsupportedFieldType(field)) {
+ return; // Ignore majority of struct fields for indexed search (only implemented for streaming search)
}
if (field.isImportedField()) {
deriveImportedAttributes(field);
- } else if (fieldIsArrayOfSimpleStruct) {
+ } else if (isArrayOfSimpleStruct(field)) {
deriveArrayOfSimpleStruct(field);
+ } else if (isMapOfSimpleStruct(field)) {
+ deriveMapOfSimpleStruct(field);
} else {
deriveAttributes(field);
}
}
+ private static boolean unsupportedFieldType(ImmutableSDField field) {
+ return (field.usesStructOrMap() &&
+ !isArrayOfSimpleStruct(field) &&
+ !isMapOfSimpleStruct(field) &&
+ !field.getDataType().equals(PositionDataType.INSTANCE) &&
+ !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)));
+ }
+
private static boolean isArrayOfSimpleStruct(ImmutableSDField field) {
DataType fieldType = field.getDataType();
if (fieldType instanceof ArrayDataType) {
ArrayDataType arrayType = (ArrayDataType)fieldType;
- DataType nestedType = arrayType.getNestedType();
- if (nestedType instanceof StructDataType &&
- !(nestedType.equals(PositionDataType.INSTANCE))) {
- StructDataType structType = (StructDataType)nestedType;
- for (Field innerField : structType.getFields()) {
- if (!isPrimitiveType(innerField.getDataType())) {
- return false;
- }
+ return isSimpleStruct(arrayType.getNestedType());
+ } else {
+ return false;
+ }
+ }
+
+ private static boolean isMapOfSimpleStruct(ImmutableSDField field) {
+ DataType fieldType = field.getDataType();
+ if (fieldType instanceof MapDataType) {
+ MapDataType mapType = (MapDataType)fieldType;
+ return isPrimitiveType(mapType.getKeyType()) &&
+ isSimpleStruct(mapType.getValueType());
+ } else {
+ return false;
+ }
+ }
+
+ private static boolean isSimpleStruct(DataType type) {
+ if (type instanceof StructDataType &&
+ !(type.equals(PositionDataType.INSTANCE))) {
+ StructDataType structType = (StructDataType) type;
+ for (Field innerField : structType.getFields()) {
+ if (!isPrimitiveType(innerField.getDataType())) {
+ return false;
}
- return true;
- } else {
- return false;
}
+ return true;
} else {
return false;
}
@@ -138,14 +159,33 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
private void deriveArrayOfSimpleStruct(ImmutableSDField field) {
for (ImmutableSDField structField : field.getStructFields()) {
- for (Attribute attribute : structField.getAttributes().values()) {
- if (structField.getName().equals(attribute.getName())) {
- attributes.put(attribute.getName(), attribute.convertToArray());
- }
+ deriveAttributesAsArrayType(structField);
+ }
+ }
+
+ private void deriveAttributesAsArrayType(ImmutableSDField field) {
+ for (Attribute attribute : field.getAttributes().values()) {
+ if (field.getName().equals(attribute.getName())) {
+ attributes.put(attribute.getName(), attribute.convertToArray());
}
}
}
+ private void deriveMapOfSimpleStruct(ImmutableSDField field) {
+ deriveMapKeyField(field.getStructField("key"));
+ deriveMapValueField(field.getStructField("value"));
+ }
+
+ private void deriveMapKeyField(ImmutableSDField keyField) {
+ deriveAttributesAsArrayType(keyField);
+ }
+
+ private void deriveMapValueField(ImmutableSDField valueField) {
+ for (ImmutableSDField structField : valueField.getStructFields()) {
+ deriveAttributesAsArrayType(structField);
+ }
+ }
+
/** Returns a read only attribute iterator */
public Iterator attributeIterator() {
return attributes().iterator();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java
index 4bdef0607a2..0661ef67131 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java
@@ -21,45 +21,11 @@ import static java.util.stream.Collectors.toSet;
public class GlobalDistributionValidator {
public void validate(Map<String, NewDocumentType> documentDefinitions,
- Set<NewDocumentType> globallyDistributedDocuments,
- Redundancy redundancy,
- boolean enableMultipleBucketSpaces) {
- if (!enableMultipleBucketSpaces) {
- verifyGlobalDocumentsHaveRequiredRedundancy(globallyDistributedDocuments, redundancy);
- verifySearchableCopiesIsSameAsRedundancy(globallyDistributedDocuments, redundancy);
- }
+ Set<NewDocumentType> globallyDistributedDocuments) {
verifyReferredDocumentsArePresent(documentDefinitions);
verifyReferredDocumentsAreGlobal(documentDefinitions, globallyDistributedDocuments);
}
- private static void verifyGlobalDocumentsHaveRequiredRedundancy(Set<NewDocumentType> globallyDistributedDocuments,
- Redundancy redundancy) {
- if (!globallyDistributedDocuments.isEmpty() && !redundancy.isEffectivelyGloballyDistributed()) {
- throw new IllegalArgumentException(
- String.format(
- "The following document types are marked as global, " +
- "but do not have high enough redundancy to make the documents globally distributed: %s. " +
- "Redundancy is %d, expected %d.",
- asPrintableString(toDocumentNameStream(globallyDistributedDocuments)),
- redundancy.effectiveFinalRedundancy(),
- redundancy.totalNodes()));
- }
- }
-
- private static void verifySearchableCopiesIsSameAsRedundancy(Set<NewDocumentType> globallyDistributedDocuments,
- Redundancy redundancy) {
- if (!globallyDistributedDocuments.isEmpty() &&
- redundancy.effectiveReadyCopies() != redundancy.effectiveFinalRedundancy()) {
- throw new IllegalArgumentException(
- String.format(
- "The following document types have the number of searchable copies less than redundancy: %s. " +
- "Searchable copies is %d, while redundancy is %d.",
- asPrintableString(toDocumentNameStream(globallyDistributedDocuments)),
- redundancy.effectiveReadyCopies(),
- redundancy.effectiveFinalRedundancy()));
- }
- }
-
private static void verifyReferredDocumentsArePresent(Map<String, NewDocumentType> documentDefinitions) {
Set<NewDocumentType.Name> unknowDocuments = getReferencedDocuments(documentDefinitions)
.filter(name -> !documentDefinitions.containsKey(name.toString()))
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 0119bced095..68ae4d2b242 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -61,14 +61,12 @@ public class ContentCluster extends AbstractConfigProducer implements
MessagetyperouteselectorpolicyConfig.Producer,
BucketspacesConfig.Producer {
- // TODO: Make private
private String documentSelection;
private ContentSearchCluster search;
private final boolean isHostedVespa;
private final Map<String, NewDocumentType> documentDefinitions;
private final Set<NewDocumentType> globallyDistributedDocuments;
- // Experimental flag (TODO: remove when feature is enabled by default)
- private boolean enableMultipleBucketSpaces = false;
+ private boolean forceEnableMultipleBucketSpaces = false;
private com.yahoo.vespa.model.content.StorageGroup rootGroup;
private StorageCluster storageNodes;
private DistributorCluster distributorNodes;
@@ -250,7 +248,7 @@ public class ContentCluster extends AbstractConfigProducer implements
private void setupExperimental(ContentCluster cluster, ModelElement experimental) {
Boolean enableMultipleBucketSpaces = experimental.childAsBoolean("enable-multiple-bucket-spaces");
if (enableMultipleBucketSpaces != null) {
- cluster.enableMultipleBucketSpaces = enableMultipleBucketSpaces;
+ cluster.forceEnableMultipleBucketSpaces = enableMultipleBucketSpaces;
}
}
@@ -596,13 +594,13 @@ public class ContentCluster extends AbstractConfigProducer implements
builder.min_distributor_up_ratio(0);
builder.min_storage_up_ratio(0);
}
- builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces);
+ builder.enable_multiple_bucket_spaces(true);
// Telling the controller whether we actually _have_ global document types lets
// it selectively enable or disable constraints that aren't needed when these
// are not are present, even if full protocol and backend support is enabled
// for multiple bucket spaces. Basically, if you don't use it, you don't
// pay for it.
- builder.cluster_has_global_document_types(enableMultipleBucketSpaces && !globallyDistributedDocuments.isEmpty());
+ builder.cluster_has_global_document_types(!globallyDistributedDocuments.isEmpty());
}
@Override
@@ -646,7 +644,7 @@ public class ContentCluster extends AbstractConfigProducer implements
}
}
new ReservedDocumentTypeNameValidator().validate(documentDefinitions);
- new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments, redundancy, enableMultipleBucketSpaces);
+ new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments);
}
public static Map<String, Integer> METRIC_INDEX_MAP = new TreeMap<>();
@@ -727,11 +725,13 @@ public class ContentCluster extends AbstractConfigProducer implements
for (NewDocumentType docType : getDocumentDefinitions().values()) {
BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder();
docTypeBuilder.name(docType.getName());
- String bucketSpace = ((enableMultipleBucketSpaces && isGloballyDistributed(docType))
- ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE);
+ String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE);
docTypeBuilder.bucketspace(bucketSpace);
builder.documenttype(docTypeBuilder);
}
- builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces);
+ // NOTE: this config is kept around to allow the use of multiple bucket spaces
+ // on older versions of Vespa. It is for all intents and purposes a no-op in
+ // newer versions where multiple bucket spaces are enabled by default.
+ builder.enable_multiple_bucket_spaces(forceEnableMultipleBucketSpaces);
}
}
diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
new file mode 100644
index 00000000000..604fc1f6ea7
--- /dev/null
+++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
@@ -0,0 +1,60 @@
+attribute[0].name "elem_map.key"
+attribute[0].datatype STRING
+attribute[0].collectiontype ARRAY
+attribute[0].removeifzero false
+attribute[0].createifnonexistent false
+attribute[0].fastsearch false
+attribute[0].huge false
+attribute[0].sortascending true
+attribute[0].sortfunction UCA
+attribute[0].sortstrength PRIMARY
+attribute[0].sortlocale ""
+attribute[0].enablebitvectors false
+attribute[0].enableonlybitvector false
+attribute[0].fastaccess false
+attribute[0].arity 8
+attribute[0].lowerbound -9223372036854775808
+attribute[0].upperbound 9223372036854775807
+attribute[0].densepostinglistthreshold 0.4
+attribute[0].tensortype ""
+attribute[0].imported false
+attribute[1].name "elem_map.value.name"
+attribute[1].datatype STRING
+attribute[1].collectiontype ARRAY
+attribute[1].removeifzero false
+attribute[1].createifnonexistent false
+attribute[1].fastsearch false
+attribute[1].huge false
+attribute[1].sortascending true
+attribute[1].sortfunction UCA
+attribute[1].sortstrength PRIMARY
+attribute[1].sortlocale ""
+attribute[1].enablebitvectors false
+attribute[1].enableonlybitvector false
+attribute[1].fastaccess false
+attribute[1].arity 8
+attribute[1].lowerbound -9223372036854775808
+attribute[1].upperbound 9223372036854775807
+attribute[1].densepostinglistthreshold 0.4
+attribute[1].tensortype ""
+attribute[1].imported false
+attribute[2].name "elem_map.value.weight"
+attribute[2].datatype INT32
+attribute[2].collectiontype ARRAY
+attribute[2].removeifzero false
+attribute[2].createifnonexistent false
+attribute[2].fastsearch false
+attribute[2].huge false
+attribute[2].sortascending true
+attribute[2].sortfunction UCA
+attribute[2].sortstrength PRIMARY
+attribute[2].sortlocale ""
+attribute[2].enablebitvectors false
+attribute[2].enableonlybitvector false
+attribute[2].fastaccess false
+attribute[2].arity 8
+attribute[2].lowerbound -9223372036854775808
+attribute[2].upperbound 9223372036854775807
+attribute[2].densepostinglistthreshold 0.4
+attribute[2].tensortype ""
+attribute[2].imported false \ No newline at end of file
diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
new file mode 100644
index 00000000000..7af49d95d09
--- /dev/null
+++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
@@ -0,0 +1,11 @@
+defaultsummaryid 653486243
+classes[0].id 653486243
+classes[0].name "default"
+classes[0].fields[0].name "elem_map"
+classes[0].fields[0].type "jsonstring"
+classes[0].fields[1].name "rankfeatures"
+classes[0].fields[1].type "featuredata"
+classes[0].fields[2].name "summaryfeatures"
+classes[0].fields[2].type "featuredata"
+classes[0].fields[3].name "documentid"
+classes[0].fields[3].type "longstring" \ No newline at end of file
diff --git a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg
new file mode 100644
index 00000000000..42b6e811ee6
--- /dev/null
+++ b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg
@@ -0,0 +1,7 @@
+defaultoutputclass -1
+override[0].field "rankfeatures"
+override[0].command "rankfeatures"
+override[0].arguments ""
+override[1].field "summaryfeatures"
+override[1].command "summaryfeatures"
+override[1].arguments "" \ No newline at end of file
diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd
new file mode 100644
index 00000000000..cb2eac4ed78
--- /dev/null
+++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd
@@ -0,0 +1,20 @@
+search test {
+ document test {
+ struct elem {
+ field name type string {}
+ field weight type int {}
+ }
+ field elem_map type map<string,elem> {
+ indexing: summary
+ struct-field key {
+ indexing: attribute
+ }
+ struct-field value.name {
+ indexing: attribute
+ }
+ struct-field value.weight {
+ indexing: attribute
+ }
+ }
+ }
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
index 990ebe7f993..c3cfcae66e6 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
@@ -79,6 +79,17 @@ public class AttributeListTestCase extends SearchDefinitionTestCase {
assertTrue(!attributes.hasNext());
}
+ @Test
+ public void map_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException {
+ Search search = SearchBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd");
+ Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator();
+
+ assertAttribute("elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next());
+ assertAttribute("elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next());
+ assertAttribute("elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next());
+ assertTrue(!attributes.hasNext());
+ }
+
private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) {
assertEquals(name, attr.getName());
assertEquals(type, attr.getType());
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
index 9e73edf9b35..72c7aab4a39 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java
@@ -18,4 +18,14 @@ public class AttributesTestCase extends AbstractExportingTestCase {
assertCorrectDeriving("attributes");
}
+ @Test
+ public void testArrayOfStructAttribute() throws IOException, ParseException {
+ assertCorrectDeriving("array_of_struct_attribute");
+ }
+
+ @Test
+ public void testMapOfStructAttribute() throws IOException, ParseException {
+ assertCorrectDeriving("map_of_struct_attribute");
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index dc2d3b7cea1..4600f6ae4c6 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -139,9 +139,4 @@ public class ExportingTestCase extends AbstractExportingTestCase {
assertCorrectDeriving("tensor");
}
- @Test
- public void testArrayOfStructAttribute() throws IOException, ParseException {
- assertCorrectDeriving("array_of_struct_attribute");
- }
-
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
index 98177b4ada0..0156128f7ca 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
@@ -173,17 +173,17 @@ public class ContentSearchClusterTest {
}
@Test
- public void require_that_all_document_types_belong_to_default_bucket_space_by_default() throws Exception {
+ public void require_that_document_types_belong_to_correct_bucket_spaces() throws Exception {
BucketspacesConfig config = getBucketspacesConfig(createClusterWithGlobalType());
assertEquals(2, config.documenttype().size());
- assertDocumentType("global", "default", config.documenttype(0));
+ assertDocumentType("global", "global", config.documenttype(0));
assertDocumentType("regular", "default", config.documenttype(1));
// Safeguard against flipping the switch
assertFalse(config.enable_multiple_bucket_spaces());
}
@Test
- public void require_that_multiple_bucket_spaces_can_be_enabled() throws Exception {
+ public void require_that_multiple_bucket_spaces_can_be_force_enabled() throws Exception {
ContentCluster cluster = createClusterWithMultipleBucketSpacesEnabled();
{
BucketspacesConfig config = getBucketspacesConfig(cluster);
@@ -210,9 +210,9 @@ public class ContentSearchClusterTest {
}
@Test
- public void controller_global_documents_config_forced_to_false_if_multiple_spaces_not_enabled() throws Exception {
+ public void controller_global_documents_config_always_enabled_even_without_experimental_flag_set() throws Exception {
ContentCluster cluster = createClusterWithGlobalDocsButNotMultipleSpacesEnabled();
- assertFalse(getFleetcontrollerConfig(cluster).cluster_has_global_document_types());
+ assertTrue(getFleetcontrollerConfig(cluster).cluster_has_global_document_types());
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java
index b8252f2f081..6506f7a08a8 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java
@@ -26,64 +26,16 @@ public class GlobalDistributionValidatorTest {
public final ExpectedException exceptionRule = ExpectedException.none();
@Test
- public void throws_exception_if_redudancy_does_not_imply_global_distribution() {
- Fixture fixture = new Fixture()
- .addGlobalDocument(createDocumentType("foo"))
- .addGlobalDocument(createDocumentType("bar"));
- Redundancy redundancy = createRedundancyWithoutGlobalDistribution();
-
- exceptionRule.expect(IllegalArgumentException.class);
- exceptionRule.expectMessage(
- "The following document types are marked as global, " +
- "but do not have high enough redundancy to make the documents globally distributed: " +
- "'bar', 'foo'. Redundancy is 2, expected 3.");
- validate(fixture, redundancy);
- }
-
- @Test
- public void validation_of_redundancy_is_deactivated_if_multiple_bucket_spaces_is_enabled() {
- Fixture fixture = new Fixture()
- .addGlobalDocument(createDocumentType("foo"))
- .addGlobalDocument(createDocumentType("bar"));
- Redundancy redundancy = createRedundancyWithoutGlobalDistributionAndTooFewSearchableCopies();
-
- validate(fixture, redundancy, true);
- }
-
- @Test
- public void throws_exception_if_searchable_copies_too_low() {
- Fixture fixture = new Fixture()
- .addGlobalDocument(createDocumentType("foo"))
- .addGlobalDocument(createDocumentType("bar"));
- Redundancy redundancy = createRedundancyWithTooFewSearchableCopies();
-
- exceptionRule.expect(IllegalArgumentException.class);
- exceptionRule.expectMessage(
- "The following document types have the number of searchable copies less than redundancy: " +
- "'bar', 'foo'. Searchable copies is 1, while redundancy is 2.");
- validate(fixture, redundancy);
- }
-
- @Test
- public void validation_succeeds_when_globally_distributed_and_enough_searchable_copies() {
- Fixture fixture = new Fixture()
- .addGlobalDocument(createDocumentType("foo"));
- Redundancy redundancy = createRedundancyWithGlobalDistribution();
- validate(fixture, redundancy);
- }
-
- @Test
public void validation_succeeds_on_no_documents() {
new GlobalDistributionValidator()
- .validate(emptyMap(), emptySet(), createRedundancyWithoutGlobalDistribution(), false);
+ .validate(emptyMap(), emptySet());
}
@Test
public void validation_succeeds_on_no_global_documents() {
Fixture fixture = new Fixture()
.addNonGlobalDocument(createDocumentType("foo"));
- Redundancy redundancy = createRedundancyWithoutGlobalDistribution();
- validate(fixture, redundancy);
+ validate(fixture);
}
@Test
@@ -92,11 +44,10 @@ public class GlobalDistributionValidatorTest {
Fixture fixture = new Fixture()
.addNonGlobalDocument(parent)
.addNonGlobalDocument(createDocumentType("child", parent));
- Redundancy redundancy = createRedundancyWithoutGlobalDistribution();
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(
"The following document types are referenced from other documents, but are not globally distributed: 'parent'");
- validate(fixture, redundancy);
+ validate(fixture);
}
@Test
@@ -105,8 +56,7 @@ public class GlobalDistributionValidatorTest {
Fixture fixture = new Fixture()
.addGlobalDocument(parent)
.addNonGlobalDocument(createDocumentType("child", parent));
- Redundancy redundancy = createRedundancyWithGlobalDistribution();
- validate(fixture, redundancy);
+ validate(fixture);
}
@Test
@@ -115,11 +65,10 @@ public class GlobalDistributionValidatorTest {
NewDocumentType child = createDocumentType("child", unknown);
Fixture fixture = new Fixture()
.addNonGlobalDocument(child);
- Redundancy redundancy = createRedundancyWithGlobalDistribution();
exceptionRule.expect(IllegalArgumentException.class);
exceptionRule.expectMessage(
"The following document types are referenced from other documents, but are not listed in services.xml: 'unknown'");
- validate(fixture, redundancy);
+ validate(fixture);
}
@Test
@@ -130,42 +79,14 @@ public class GlobalDistributionValidatorTest {
new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/global_distribution_validation/").create();
}
- private static Redundancy createRedundancyWithGlobalDistribution() {
- Redundancy redundancy = new Redundancy(2, 2, 2);
- redundancy.setTotalNodes(2);
- return redundancy;
- }
-
- private static Redundancy createRedundancyWithoutGlobalDistribution() {
- Redundancy redundancy = new Redundancy(2, 2, 2);
- redundancy.setTotalNodes(3);
- return redundancy;
- }
-
- private static Redundancy createRedundancyWithTooFewSearchableCopies() {
- Redundancy redundancy = new Redundancy(2, 2, 1);
- redundancy.setTotalNodes(2);
- return redundancy;
- }
-
- private static Redundancy createRedundancyWithoutGlobalDistributionAndTooFewSearchableCopies() {
- Redundancy redundancy = new Redundancy(2, 2, 1);
- redundancy.setTotalNodes(3);
- return redundancy;
- }
-
private static NewDocumentType createDocumentType(String name, NewDocumentType... references) {
Set<NewDocumentType.Name> documentReferences = Stream.of(references).map(NewDocumentType::getFullName).collect(toSet());
return new NewDocumentType(new NewDocumentType.Name(name), documentReferences);
}
- private static void validate(Fixture fixture, Redundancy redundancy) {
- validate(fixture, redundancy, false);
- }
-
- private static void validate(Fixture fixture, Redundancy redundancy, boolean enableMultipleBucketSpaces) {
+ private static void validate(Fixture fixture) {
new GlobalDistributionValidator()
- .validate(fixture.getDocumentTypes(), fixture.getGloballyDistributedDocuments(), redundancy, enableMultipleBucketSpaces);
+ .validate(fixture.getDocumentTypes(), fixture.getGloballyDistributedDocuments());
}
private static class Fixture {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
index eabd0e5a7e0..131a5344116 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
@@ -7,10 +7,10 @@ import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.model.test.MockHosts;
import org.junit.Test;
+import java.io.File;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
-import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -48,5 +48,10 @@ public class FileDistributorTestCase {
public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {
filesToDownloadCalled++;
}
+
+ @Override
+ public File getFileReferencesDir() {
+ return null;
+ }
}
}
diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java
new file mode 100644
index 00000000000..8c0e276b3d3
--- /dev/null
+++ b/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.athenz.instanceproviderservice.config;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def
index 281db6fb43d..281db6fb43d 100644
--- a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def
+++ b/configdefinitions/src/vespa/athenz-provider-service.def
diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def
index 5e81526dc53..bf4c9599f4a 100644
--- a/configdefinitions/src/vespa/configserver.def
+++ b/configdefinitions/src/vespa/configserver.def
@@ -13,10 +13,13 @@ zookeeperserver[].port int default=2181
zookeeper.barrierTimeout long default=120 # in seconds
zookeeperLocalhostAffinity bool default=true
-# Misc
+# Directories
configModelPluginDir[] string
configServerDBDir string default="var/db/vespa/config_server/serverdb/"
configDefinitionsDir string default="share/vespa/configdefinitions/"
+fileReferencesDir string default="var/db/vespa/filedistribution/"
+
+# Misc
sessionLifetime long default=3600 # in seconds
masterGeneration long default=0
multitenant bool default=false
diff --git a/configgen/pom.xml b/configgen/pom.xml
index e8de114f8d5..be9cd733601 100644
--- a/configgen/pom.xml
+++ b/configgen/pom.xml
@@ -19,14 +19,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-xml_${scala.major-version}</artifactId>
- </dependency>
</dependencies>
<build>
<plugins>
@@ -63,32 +55,6 @@
<updateReleaseInfo>true</updateReleaseInfo>
</configuration>
</plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>compile</id>
- <goals>
- <goal>compile</goal>
- </goals>
- <phase>compile</phase>
- </execution>
- <execution>
- <id>test-compile</id>
- <goals>
- <goal>testCompile</goal>
- </goals>
- <phase>test-compile</phase>
- </execution>
- <execution>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</project>
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java
new file mode 100644
index 00000000000..bf3fc2902a1
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java
@@ -0,0 +1,351 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import com.yahoo.config.codegen.LeafCNode.FileLeaf;
+import com.yahoo.config.codegen.LeafCNode.PathLeaf;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static com.yahoo.config.codegen.ConfigGenerator.boxedDataType;
+import static com.yahoo.config.codegen.ConfigGenerator.indentCode;
+import static com.yahoo.config.codegen.ConfigGenerator.nodeClass;
+import static com.yahoo.config.codegen.ConfigGenerator.userDataType;
+import static com.yahoo.config.codegen.JavaClassBuilder.INDENTATION;
+import static com.yahoo.config.codegen.JavaClassBuilder.createUniqueSymbol;
+import static com.yahoo.config.codegen.ReservedWords.INTERNAL_PREFIX;
+import static java.util.Arrays.stream;
+
+/**
+ * @author gjoranv
+ * @author ollivir
+ */
+
+public class BuilderGenerator {
+ public static String getBuilder(InnerCNode node) {
+ return getDeclaration(node) + "\n" + //
+ indentCode(INDENTATION, getUninitializedScalars(node) + "\n\n" + //
+ stream(node.getChildren()).map(BuilderGenerator::getBuilderFieldDefinition).collect(Collectors.joining("\n"))
+ + "\n\n" + //
+ getBuilderConstructors(node, nodeClass(node)) + "\n\n" + //
+ getOverrideMethod(node) + "\n\n" + //
+ getBuilderSetters(node) + "\n" + //
+ getSpecialRootBuilderCode(node))
+ + "}";
+ }
+
+ private static String getDeclaration(InnerCNode node) {
+ String getInterfaces = (node.getParent() == null) ? "implements ConfigInstance.Builder" : "implements ConfigBuilder";
+
+ return "public static class Builder " + getInterfaces + " {";
+ }
+
+ private static String getSpecialRootBuilderCode(InnerCNode node) {
+ return (node.getParent() == null) ? "\n" + getDispatchCode() + "\n" : "";
+ }
+
+ private static String getDispatchCode() {
+ // Use full path to @Override, as users are free to define an inner node called
+ // 'override'. (summarymap.def does)
+ // The generated inner 'Override' class would otherwise be mistaken for the
+ // annotation.
+ return "@java.lang.Override\n" + //
+ "public final boolean dispatchGetConfig(ConfigInstance.Producer producer) {\n" + //
+ " if (producer instanceof Producer) {\n" + //
+ " ((Producer)producer).getConfig(this);\n" + //
+ " return true;\n" + //
+ " }\n" + //
+ " return false;\n" + //
+ "}\n" + //
+ "\n" + //
+ "@java.lang.Override\n" + //
+ "public final String getDefMd5() { return CONFIG_DEF_MD5; }\n" + //
+ "@java.lang.Override\n" + //
+ "public final String getDefName() { return CONFIG_DEF_NAME; }\n" + //
+ "@java.lang.Override\n" + //
+ "public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }";
+ }
+
+ private static String getUninitializedScalars(InnerCNode node) {
+ List<String> scalarsWithoutDefault = new ArrayList<>();
+ for (CNode child : node.getChildren()) {
+ if (child instanceof LeafCNode && (!child.isArray && !child.isMap && ((LeafCNode) child).getDefaultValue() == null)) {
+ scalarsWithoutDefault.add("\"" + child.getName() + "\"");
+ }
+ }
+
+ String uninitializedList = (scalarsWithoutDefault.size() > 0)
+ ? "Arrays.asList(\n" + indentCode(INDENTATION, String.join(",\n", scalarsWithoutDefault) + "\n)")
+ : "";
+
+ return "private Set<String> " + INTERNAL_PREFIX + "uninitialized = new HashSet<String>(" + uninitializedList + ");";
+ }
+
+ private static String getBuilderFieldDefinition(CNode node) {
+ if (node.isArray) {
+ return String.format("public List<%s> %s = new ArrayList<>();", builderType(node), node.getName());
+ } else if (node.isMap) {
+ return String.format("public Map<String, %s> %s = new LinkedHashMap<>();", builderType(node), node.getName());
+ } else if (node instanceof InnerCNode) {
+ return String.format("public %s %s = new %s();", builderType(node), node.getName(), builderType(node));
+ } else if (node instanceof LeafCNode) {
+ return String.format("private %s %s = null;", boxedBuilderType((LeafCNode) node), node.getName());
+ } else {
+ throw new IllegalStateException("Cannot produce builder field definition for node"); // Should not happen
+ }
+ }
+
+ private static String getBuilderSetters(CNode node) {
+ List<String> elem = new ArrayList<>();
+ CNode[] children = node.getChildren();
+
+ for (CNode child : children) {
+ if (child instanceof InnerCNode && child.isArray) {
+ elem.add(BuilderSetters.innerArraySetters((InnerCNode) child));
+ } else if (child instanceof InnerCNode && child.isMap) {
+ elem.add(BuilderSetters.innerMapSetters(child));
+ } else if (child instanceof LeafCNode && child.isArray) {
+ elem.add(BuilderSetters.leafArraySetters((LeafCNode) child));
+ } else if (child instanceof LeafCNode && child.isMap) {
+ elem.add(BuilderSetters.leafMapSetters(child));
+ } else if (child instanceof InnerCNode) {
+ elem.add(BuilderSetters.structSetter((InnerCNode) child));
+ } else if (child instanceof LeafCNode) {
+ elem.add(BuilderSetters.scalarSetters((LeafCNode) child));
+ }
+ }
+ return String.join("\n\n", elem);
+ }
+
+ private static class BuilderSetters {
+ private static String structSetter(InnerCNode n) {
+ return "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "builder) {\n" + //
+ " " + n.getName() + " = " + INTERNAL_PREFIX + "builder;\n" + //
+ " return this;\n" + //
+ "}";
+ }
+
+ private static String innerArraySetters(InnerCNode n) {
+ return "/**\n" + //
+ " * Add the given builder to this builder's list of " + nodeClass(n) + " builders\n" + //
+ " * @param " + INTERNAL_PREFIX + "builder a builder\n" + //
+ " * @return this builder\n" + //
+ " */\n" + //
+ "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "builder) {\n" + //
+ " " + n.getName() + ".add(" + INTERNAL_PREFIX + "builder);\n" + //
+ " return this;\n" + //
+ "}\n" + //
+ "\n" + //
+ "/**\n" + //
+ " * Set the given list as this builder's list of " + nodeClass(n) + " builders\n" + //
+ " * @param __builders a list of builders\n" + //
+ " * @return this builder\n" + //
+ " */\n" + //
+ "public Builder " + n.getName() + "(List<" + builderType(n) + "> __builders) {\n" + //
+ " " + n.getName() + " = __builders;\n" + //
+ " return this;\n" + //
+ "}";
+ }
+
+ private static String publicLeafNodeSetters(LeafCNode n) {
+ return "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "value) {\n" + //
+ " " + n.getName() + ".add(" + INTERNAL_PREFIX + "value);\n" + //
+ " return this;\n" + //
+ "}\n" + //
+ "\n" + //
+ "public Builder " + n.getName() + "(Collection<" + builderType(n) + "> " + INTERNAL_PREFIX + "values) {\n" + //
+ " " + n.getName() + ".addAll(" + INTERNAL_PREFIX + "values);\n" + //
+ " return this;\n" + //
+ "}";
+ }
+
+ private static String privateLeafNodeSetter(LeafCNode n) {
+ if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n))) {
+ return "";
+ } else {
+ return "\n\n" + //
+ "private Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "value) {\n" + //
+ " return " + n.getName() + "(" + builderType(n) + ".valueOf(" + INTERNAL_PREFIX + "value));\n" + //
+ "}";
+ }
+ }
+
+ private static String leafArraySetters(LeafCNode n) {
+ return publicLeafNodeSetters(n) + privateLeafNodeSetter(n);
+ }
+
+ private static String innerMapSetters(CNode n) {
+ return "public Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "key, " + builderType(n) + " " + INTERNAL_PREFIX
+ + "value) {\n" + //
+ " " + n.getName() + ".put(" + INTERNAL_PREFIX + "key, " + INTERNAL_PREFIX + "value);\n" + //
+ " return this;\n" + //
+ "}\n" + //
+ "\n" + //
+ "public Builder " + n.getName() + "(Map<String, " + builderType(n) + "> " + INTERNAL_PREFIX + "values) {\n" + //
+ " " + n.getName() + ".putAll(" + INTERNAL_PREFIX + "values);\n" + //
+ " return this;\n" + //
+ "}";
+ }
+
+ private static String privateLeafMapSetter(CNode n) {
+ if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n))) {
+ return "";
+ } else {
+ return "\n\n" + //
+ "private Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "key, String " + INTERNAL_PREFIX + "value) {\n" + //
+ " return " + n.getName() + "(" + INTERNAL_PREFIX + "key, " + builderType(n) + ".valueOf(" + INTERNAL_PREFIX
+ + "value));\n" + //
+ "}";
+ }
+ }
+
+ private static String leafMapSetters(CNode n) {
+ return innerMapSetters(n) + privateLeafMapSetter(n);
+ }
+
+ private static String scalarSetters(LeafCNode n) {
+ String name = n.getName();
+
+ String signalInitialized = (n.getDefaultValue() == null) ? " " + INTERNAL_PREFIX + "uninitialized.remove(\"" + name + "\");\n"
+ : "";
+
+ String bType = builderType(n);
+ String stringSetter = "String".equals(bType) || "FileReference".equals(bType) ? ""
+ : String.format("\nprivate Builder %s(String %svalue) {\n" + //
+ " return %s(%s.valueOf(%svalue));\n" + //
+ "}", name, INTERNAL_PREFIX, name, boxedDataType(n), INTERNAL_PREFIX);
+
+ String getNullGuard = bType.equals(boxedBuilderType(n)) ? String.format(
+ "\nif (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");", INTERNAL_PREFIX) : "";
+
+ return String.format("public Builder %s(%s %svalue) {%s\n" + //
+ " %s = %svalue;\n" + //
+ "%s", name, bType, INTERNAL_PREFIX, getNullGuard, name, INTERNAL_PREFIX, signalInitialized) + //
+ " return this;" + "\n}\n" + stringSetter;
+ }
+ }
+
+ private static String setBuilderValueFromConfig(CNode child, CNode node) {
+ final String name = child.getName();
+ final boolean isArray = child.isArray;
+ final boolean isMap = child.isMap;
+
+ if (child instanceof FileLeaf && isArray) {
+ return name + "(" + userDataType(child) + ".toValues(config." + name + "()));";
+ } else if (child instanceof FileLeaf && isMap) {
+ return name + "(" + userDataType(child) + ".toValueMap(config." + name + "()));";
+ } else if (child instanceof FileLeaf) {
+ return name + "(config." + name + "().value());";
+ } else if (child instanceof PathLeaf && isArray) {
+ return name + "(" + nodeClass(child) + ".toFileReferences(config." + name + "));";
+ } else if (child instanceof PathLeaf && isMap) {
+ return name + "(" + nodeClass(child) + ".toFileReferenceMap(config." + name + "));";
+ } else if (child instanceof PathLeaf) {
+ return name + "(config." + name + ".getFileReference());";
+ } else if (child instanceof LeafCNode) {
+ return name + "(config." + name + "());";
+ } else if (child instanceof InnerCNode && isArray) {
+ return setInnerArrayBuildersFromConfig((InnerCNode) child, node);
+ } else if (child instanceof InnerCNode && isMap) {
+ return setInnerMapBuildersFromConfig((InnerCNode) child);
+ } else {
+ return name + "(new " + builderType(child) + "(config." + name + "()));";
+ }
+ }
+
+ private static String setInnerArrayBuildersFromConfig(InnerCNode innerArr, CNode node) {
+ final String elemName = createUniqueSymbol(node, innerArr.getName());
+
+ return "for (" + userDataType(innerArr) + " " + elemName + " : config." + innerArr.getName() + "()) {\n" + //
+ " " + innerArr.getName() + "(new " + builderType(innerArr) + "(" + elemName + "));\n" + //
+ "}";
+ }
+
+ private static String setInnerMapBuildersFromConfig(InnerCNode innerMap) {
+ final String entryName = INTERNAL_PREFIX + "entry";
+ return "for (Map.Entry<String, " + userDataType(innerMap) + "> " + entryName + " : config." + innerMap.getName()
+ + "().entrySet()) {\n" + //
+ " " + innerMap.getName() + "(" + entryName + ".getKey(), new " + userDataType(innerMap) + ".Builder(" + entryName
+ + ".getValue()));\n" + //
+ "}";
+ }
+
+ private static String getBuilderConstructors(CNode node, String className) {
+ return "public Builder() { }\n" + //
+ "\n" + //
+ "public Builder(" + className + " config) {\n" + //
+ indentCode(INDENTATION,
+ stream(node.getChildren()).map(child -> setBuilderValueFromConfig(child, node)).collect(Collectors.joining("\n")))
+ + //
+ "\n}";
+ }
+
+ private static String conditionStatement(CNode child) {
+ final String superior = INTERNAL_PREFIX + "superior";
+
+ if (child.isArray) {
+ return "if (!" + superior + "." + child.getName() + ".isEmpty())";
+ } else if (child.isMap) {
+ return "";
+ } else if (child instanceof LeafCNode) {
+ return "if (" + superior + "." + child.getName() + " != null)";
+ } else {
+ return "";
+ }
+ }
+
+ private static String overrideBuilderValue(CNode child) {
+ final String superior = INTERNAL_PREFIX + "superior";
+ final String method = "override";
+ final String name = child.getName();
+ final String callSetter = name + "(" + superior + "." + name + ");";
+
+ if (child.isArray) {
+ String arrayOverride = INDENTATION + name + ".addAll(" + superior + "." + name + ");";
+ return conditionStatement(child) + "\n" + arrayOverride;
+ } else if (child instanceof InnerCNode && !child.isArray && !child.isMap) {
+ return name + "(" + name + "." + method + "(" + superior + "." + name + "));";
+ } else if (child.isMap) {
+ return callSetter;
+ } else {
+ return conditionStatement(child) + "\n" + INDENTATION + callSetter;
+ }
+ }
+
+ private static String getOverrideMethod(CNode node) {
+ final String superior = INTERNAL_PREFIX + "superior";
+ final String method = "override";
+
+ return "private Builder " + method + "(Builder " + superior + ") {\n" + //
+ indentCode(INDENTATION,
+ stream(node.getChildren()).map(BuilderGenerator::overrideBuilderValue).collect(Collectors.joining("\n")))
+ + "\n" + //
+ " return this;\n" + //
+ "}";
+ }
+
+ private static String builderType(CNode node) {
+ if (node instanceof InnerCNode) {
+ return boxedDataType(node) + ".Builder";
+ } else if (node instanceof FileLeaf) {
+ return "String";
+ } else if (node instanceof PathLeaf) {
+ return "FileReference";
+ } else if (node instanceof LeafCNode && (node.isArray || node.isMap)) {
+ return boxedDataType(node);
+ } else {
+ return userDataType(node);
+ }
+ }
+
+ private static String boxedBuilderType(LeafCNode node) {
+ if (node instanceof FileLeaf) {
+ return "String";
+ } else if (node instanceof PathLeaf) {
+ return "FileReference";
+ } else {
+ return boxedDataType(node);
+ }
+ }
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java
new file mode 100644
index 00000000000..9980cf565b1
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java
@@ -0,0 +1,444 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import com.yahoo.config.codegen.LeafCNode.BooleanLeaf;
+import com.yahoo.config.codegen.LeafCNode.DoubleLeaf;
+import com.yahoo.config.codegen.LeafCNode.EnumLeaf;
+import com.yahoo.config.codegen.LeafCNode.FileLeaf;
+import com.yahoo.config.codegen.LeafCNode.IntegerLeaf;
+import com.yahoo.config.codegen.LeafCNode.LongLeaf;
+import com.yahoo.config.codegen.LeafCNode.PathLeaf;
+import com.yahoo.config.codegen.LeafCNode.ReferenceLeaf;
+import com.yahoo.config.codegen.LeafCNode.StringLeaf;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static com.yahoo.config.codegen.BuilderGenerator.getBuilder;
+import static com.yahoo.config.codegen.JavaClassBuilder.INDENTATION;
+import static com.yahoo.config.codegen.ReservedWords.INTERNAL_PREFIX;
+import static java.util.Arrays.stream;
+
+/**
+ * @author gjoranv
+ * @author Tony Vaagenes
+ * @author ollivir
+ */
+public class ConfigGenerator {
+ // TODO: don't take indent as method param - the caller should indent
+ public static String generateContent(String indent, InnerCNode node, boolean isOuter) {
+ CNode[] children = node.getChildren();
+
+ return indentCode(indent,
+ getBuilder(node) + "\n\n" +
+ stream(children).map(ConfigGenerator::getFieldDefinition).collect(Collectors.joining("\n")) + "\n\n" +
+ getConstructors(node) + "\n\n" +
+ getAccessors(children) + "\n\n" +
+ getGetChangesRequiringRestart(node) + "\n\n" +
+ getContainsFieldsFlaggedWithRestart(node, isOuter) +
+ getStaticMethods(node) +
+ generateCodeForChildren(children, indent)
+ );
+ }
+
+ private static String generateCodeForChildren(CNode[] children, String indent) {
+ List<String> pieces = new LinkedList<>();
+ for (CNode child : children) {
+ if (child instanceof EnumLeaf) {
+ pieces.add(getEnumCode((EnumLeaf) child) + "\n");
+ } else if (child instanceof InnerCNode) {
+ pieces.add(getInnerDefinition((InnerCNode) child, indent) + "\n");
+ }
+ }
+ return String.join("\n", pieces);
+ }
+
+ private static String getInnerDefinition(InnerCNode inner, String indent) {
+ return (getClassDoc(inner) + "\n" +//
+ getClassDeclaration(inner) + "\n" +//
+ generateContent(indent, inner, false)).trim() + "\n}";
+ }
+
+ private static String getClassDeclaration(CNode node) {
+ return "public final static class " + nodeClass(node) + " extends InnerNode { \n";
+ }
+
+ private static String getFieldDefinition(CNode node) {
+ String fieldDef;
+ if (node instanceof LeafCNode && node.isArray) {
+ fieldDef = String.format("LeafNodeVector<%s, %s> %s;", boxedDataType(node), nodeClass(node), node.getName());
+ } else if (node instanceof InnerCNode && node.isArray) {
+ fieldDef = String.format("InnerNodeVector<%s> %s;", nodeClass(node), node.getName());
+ } else if (node.isMap) {
+ fieldDef = String.format("Map<String, %s> %s;", nodeClass(node), node.getName());
+ } else {
+ fieldDef = String.format("%s %s;", nodeClass(node), node.getName());
+ }
+ return node.getCommentBlock("//") + "private final " + fieldDef;
+ }
+
+ private static String getStaticMethods(InnerCNode node) {
+ if (node.isArray) {
+ return getStaticMethodsForInnerArray(node) + "\n\n";
+ } else if (node.isMap) {
+ return getStaticMethodsForInnerMap(node) + "\n\n";
+ } else {
+ return "";
+ }
+ }
+
+ private static String getContainsFieldsFlaggedWithRestart(CNode node, boolean isOuter) {
+ if (isOuter) {
+ return String.format("private static boolean containsFieldsFlaggedWithRestart() {\n" +//
+ " return %b;\n" +//
+ "}\n\n", node.needRestart());
+ } else {
+ return "";
+ }
+ }
+
+ private static String getGetChangesRequiringRestart(InnerCNode node) {
+ List<String> comparisons = new LinkedList<>();
+ for (CNode child : node.getChildren()) {
+ if (child.needRestart()) {
+ comparisons.add("\n " + getComparison(child));
+ }
+ }
+
+ return "private ChangesRequiringRestart getChangesRequiringRestart(" + nodeClass(node) + " newConfig) {\n" +//
+ " ChangesRequiringRestart changes = new ChangesRequiringRestart(\"" + node.getName() + "\");" + String.join("", comparisons) + "\n" +//
+ " return changes;\n" +//
+ "}";
+ }
+
+ private static String quotedComment(CNode node) {
+ return node.getComment().replace("\n", "\\n").replace("\"", "\\\"");
+ }
+
+ private static String getComparison(CNode node) {
+ if (node instanceof InnerCNode && node.isArray) {
+ return " changes.compareArray(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +//
+ " (a,b) -> ((" + nodeClass(node) + ")a).getChangesRequiringRestart((" + nodeClass(node) + ")b));";
+ } else if (node instanceof InnerCNode && node.isMap) {
+ return " changes.compareMap(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +//
+ " (a,b) -> ((" + nodeClass(node) + ")a).getChangesRequiringRestart((" + nodeClass(node) + ")b));";
+ } else if (node instanceof InnerCNode) {
+ return " changes.mergeChanges(\"" + node.getName() + "\", this." + node.getName() + ".getChangesRequiringRestart(newConfig." + node.getName() + "));";
+ } else if (node.isArray) {
+ return " changes.compareArray(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +//
+ " (a,b) -> new ChangesRequiringRestart(\"" + node.getName() + "\").compare(a,b,\"\",\"" + quotedComment(node) + "\"));";
+ } else if (node.isMap) {
+ return " changes.compareMap(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +//
+ " (a,b) -> new ChangesRequiringRestart(\"" + node.getName() + "\").compare(a,b,\"\",\"" + quotedComment(node) + "\"));";
+ } else {
+ return " changes.compare(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\");";
+ }
+ }
+
+ private static String scalarDefault(LeafCNode scalar) {
+ if (scalar.getDefaultValue() == null) {
+ return "";
+ } else if (scalar instanceof EnumLeaf && scalar.getDefaultValue().getValue() == null) {
+ return "";
+ } else if (scalar instanceof EnumLeaf) {
+ return nodeClass(scalar) + "." + scalar.getDefaultValue().getStringRepresentation();
+ } else if (scalar instanceof LongLeaf) {
+ return scalar.getDefaultValue().getStringRepresentation() + "L";
+ } else if (scalar instanceof DoubleLeaf) {
+ return scalar.getDefaultValue().getStringRepresentation() + "D";
+ } else {
+ return scalar.getDefaultValue().getStringRepresentation();
+ }
+ }
+
+ private static String assignFromBuilder(CNode child) {
+ final String name = child.getName();
+ final String className = nodeClass(child);
+ final boolean isArray = child.isArray;
+ final boolean isMap = child.isMap;
+
+ if (child instanceof FileLeaf && isArray) {
+ return name + " = LeafNodeVector.createFileNodeVector(builder." + name + ");";
+ } else if (child instanceof PathLeaf && isArray) {
+ return name + " = LeafNodeVector.createPathNodeVector(builder." + name + ");";
+ } else if (child instanceof LeafCNode && isArray) {
+ return name + " = new LeafNodeVector<>(builder." + name + ", new " + className + "());";
+ } else if (child instanceof FileLeaf && isMap) {
+ return name + " = LeafNodeMaps.asFileNodeMap(builder." + name + ");";
+ } else if (child instanceof PathLeaf && isMap) {
+ return name + " = LeafNodeMaps.asPathNodeMap(builder." + name + ");";
+ } else if (child instanceof LeafCNode && isMap) {
+ return name + " = LeafNodeMaps.asNodeMap(builder." + name + ", new " + className + "());";
+ } else if (child instanceof InnerCNode && isArray) {
+ return name + " = " + className + ".createVector(builder." + name + ");";
+ } else if (child instanceof InnerCNode && isMap) {
+ return name + " = " + className + ".createMap(builder." + name + ");";
+ } else if (child instanceof InnerCNode) {
+ return name + " = new " + className + "(builder." + name + ", throwIfUninitialized);";
+ } else if (child instanceof LeafCNode) {
+ return name + " = (builder." + name + " == null) ?\n" +//
+ " new " + className + "(" + scalarDefault((LeafCNode) child) + ") : new " + className + "(builder." + name + ");";
+ } else {
+ throw new IllegalStateException("Cannot create assignment for node"); // should not happen
+ }
+ }
+
+ private static String getConstructors(InnerCNode inner) {
+ // TODO: merge these two constructors into one when the config library uses builders to set values from payload.
+ return "public " + nodeClass(inner) + "(Builder builder) {\n" +//
+ " this(builder, true);\n" +//
+ "}\n" +//
+ "\n" +//
+ "private " + nodeClass(inner) + "(Builder builder, boolean throwIfUninitialized) {\n" +//
+ " if (throwIfUninitialized && ! builder." + INTERNAL_PREFIX + "uninitialized.isEmpty())\n" +//
+ " throw new IllegalArgumentException(\"The following builder parameters for \" +\n" +//
+ " \"" + inner.getFullName() + " must be initialized: \" + builder." + INTERNAL_PREFIX + "uninitialized);\n" +//
+ "\n" +//
+ indentCode(INDENTATION, stream(inner.getChildren()).map(ConfigGenerator::assignFromBuilder).collect(Collectors.joining("\n"))) + "\n" +//
+ "}";
+ }
+
+ private static String getAccessorCode(CNode node) {
+ if (node.isArray) {
+ return accessorsForArray(node);
+ } else if (node.isMap) {
+ return accessorsForMap(node);
+ } else {
+ return accessorForStructOrScalar(node);
+ }
+ }
+
+ private static String valueAccessor(CNode node) {
+ if (node instanceof LeafCNode) {
+ return ".value()";
+ } else {
+ return "";
+ }
+ }
+
+ private static String listAccessor(CNode node) {
+ if (node instanceof LeafCNode) {
+ return node.getName() + ".asList()";
+ } else {
+ return node.getName();
+ }
+ }
+
+ private static String mapAccessor(CNode node) {
+ if (node instanceof LeafCNode) {
+ return "LeafNodeMaps.asValueMap(" + node.getName() + ")";
+ } else {
+ return "Collections.unmodifiableMap(" + node.getName() + ")";
+ }
+ }
+
+ private static String accessorsForArray(CNode node) {
+ final String name = node.getName();
+ final String fullName = node.getFullName();
+ return "/**\n" +//
+ " * @return " + fullName + "\n" +//
+ " */\n" +//
+ "public List<" + boxedDataType(node) + "> " + name + "() {\n" +//
+ " return " + listAccessor(node) + ";\n" +//
+ "}\n" +//
+ "\n" +//
+ "/**\n" +//
+ " * @param i the index of the value to return\n" +//
+ " * @return " + fullName + "\n" +//
+ " */\n" +//
+ "public " + userDataType(node) + " " + name + "(int i) {\n" +//
+ " return " + name + ".get(i)" + valueAccessor(node) + ";\n" +//
+ "}";
+ }
+
+ private static String accessorsForMap(CNode node) {
+ final String name = node.getName();
+ final String fullName = node.getFullName();
+
+ return "/**\n" +//
+ " * @return " + fullName + "\n" +//
+ " */\n" +//
+ "public Map<String, " + boxedDataType(node) + "> " + name + "() {\n" +//
+ " return " + mapAccessor(node) + ";\n" +//
+ "}\n" +//
+ "\n" +//
+ "/**\n" +//
+ " * @param key the key of the value to return\n" +//
+ " * @return " + fullName + "\n" +//
+ " */\n" +//
+ "public " + userDataType(node) + " " + name + "(String key) {\n" +//
+ " return " + name + ".get(key)" + valueAccessor(node) + ";\n" +//
+ "}";
+ }
+
+ private static String accessorForStructOrScalar(CNode node) {
+ return "/**\n" +//
+ " * @return " + node.getFullName() + "\n" +//
+ " */\n" +//
+ "public " + userDataType(node) + " " + node.getName() + "() {\n" +//
+ " return " + node.getName() + valueAccessor(node) + ";\n" +//
+ "}";
+ }
+
+ private static String getAccessors(CNode[] children) {
+ List<String> accessors = new LinkedList<>();
+ for (CNode child : children) {
+ String accessor = getAccessorCode(child);
+ if (accessor.isEmpty() == false) {
+ accessors.add(accessor);
+ }
+ }
+ return String.join("\n\n", accessors);
+ }
+
+ private static String getStaticMethodsForInnerArray(InnerCNode inner) {
+ final String nc = nodeClass(inner);
+ return String.format("private static InnerNodeVector<%s> createVector(List<Builder> builders) {\n" +//
+ " List<%s> elems = new ArrayList<>();\n" +//
+ " for (Builder b : builders) {\n" +//
+ " elems.add(new %s(b));\n" +//
+ " }\n" +//
+ " return new InnerNodeVector<%s>(elems);\n" +//
+ "}", nc, nc, nc, nc);
+ }
+
+ private static String getStaticMethodsForInnerMap(InnerCNode inner) {
+ final String nc = nodeClass(inner);
+ return String.format(
+ "private static Map<String, %s> createMap(Map<String, Builder> builders) {\n" +//
+ " Map<String, %s> ret = new LinkedHashMap<>();\n" +//
+ " for(String key : builders.keySet()) {\n" +//
+ " ret.put(key, new %s(builders.get(key)));\n" +//
+ " }\n" +//
+ " return Collections.unmodifiableMap(ret);\n" +//
+ "}", nc, nc, nc);
+ }
+
+ private static String getEnumCode(EnumLeaf en) {
+ String enumValues = stream(en.getLegalValues()).map(e -> String.format(" public final static Enum %s = Enum.%s;", e, e)).collect(Collectors.joining("\n"));
+
+ String code = String.format("%s\n" +//
+ "public final static class %s extends EnumNode<%s> {\n" +//
+ "\n" +//
+ " public %s(){\n" +//
+ " this.value = null;\n" +//
+ " }\n" +//
+ "\n" +//
+ " public %s(Enum enumValue) {\n" +//
+ " super(enumValue != null);\n" +//
+ " this.value = enumValue;\n" +//
+ " }\n" +//
+ "\n" +//
+ " public enum Enum {%s}\n" +//
+ "%s\n" +//
+ "\n" +//
+ " @Override\n" +//
+ " protected boolean doSetValue(@NonNull String name) {\n" +//
+ " try {\n" +//
+ " value = Enum.valueOf(name);\n" +//
+ " return true;\n" +//
+ " } catch (IllegalArgumentException e) {\n" +//
+ " }\n" +//
+ " return false;\n" +//
+ " }\n" +//
+ "}", getClassDoc(en),
+ nodeClass(en),
+ nodeClass(en) + ".Enum",
+ nodeClass(en),
+ nodeClass(en),
+ String.join(", ", en.getLegalValues()),
+ enumValues);
+
+ return indentCode("", code);
+ }
+
+ private static String getClassDoc(CNode node) {
+ String header = "/**\n" + " * This class represents " + node.getFullName();
+ String nodeComment = node.getCommentBlock(" *");
+ if (nodeComment.isEmpty()) {
+ return header + "\n */";
+ } else {
+ if (nodeComment.endsWith("\n")) {
+ nodeComment = nodeComment.substring(0, nodeComment.length() - 1);
+ }
+ return header + "\n * \n" + nodeComment + "\n */";
+ }
+ }
+
+ static String indentCode(String indent, String code) {
+ List<String> indented = new LinkedList<>();
+ for (String line : code.split("\n", -1)) {
+ indented.add(line.length() > 0 ? indent + line : line);
+ }
+ return String.join("\n", indented);
+ }
+
+ /**
+ * @return the name of the class that is generated by this node.
+ */
+ static String nodeClass(CNode node) {
+ if (node.getName().length() == 0) {
+ throw new CodegenRuntimeException("Node with empty name, under parent " + node.getParent().getName());
+ } else if (node instanceof InnerCNode && node.getParent() == null) {
+ return ConfiggenUtil.createClassName(node.getName());
+ } else if (node instanceof BooleanLeaf) {
+ return "BooleanNode";
+ } else if (node instanceof DoubleLeaf) {
+ return "DoubleNode";
+ } else if (node instanceof FileLeaf) {
+ return "FileNode";
+ } else if (node instanceof PathLeaf) {
+ return "PathNode";
+ } else if (node instanceof IntegerLeaf) {
+ return "IntegerNode";
+ } else if (node instanceof LongLeaf) {
+ return "LongNode";
+ } else if (node instanceof ReferenceLeaf) {
+ return "ReferenceNode";
+ } else if (node instanceof StringLeaf) {
+ return "StringNode";
+ } else {
+ return ConfiggenUtil.capitalize(node.getName());
+ }
+ }
+
+ static String userDataType(CNode node) {
+ if (node instanceof InnerCNode) {
+ return nodeClass(node);
+ } else if (node instanceof EnumLeaf) {
+ return nodeClass(node) + ".Enum";
+ } else if (node instanceof BooleanLeaf) {
+ return "boolean";
+ } else if (node instanceof DoubleLeaf) {
+ return "double";
+ } else if (node instanceof FileLeaf) {
+ return "FileReference";
+ } else if (node instanceof PathLeaf) {
+ return "Path";
+ } else if (node instanceof IntegerLeaf) {
+ return "int";
+ } else if (node instanceof LongLeaf) {
+ return "long";
+ } else if (node instanceof StringLeaf) {
+ return "String";
+ } else {
+ throw new IllegalStateException("Cannot determine user data type for node"); // should not occur
+ }
+ }
+
+ /**
+ * @return the boxed java data type, e.g. Integer for int
+ */
+ static String boxedDataType(CNode node) {
+ String rawType = userDataType(node);
+
+ if ("int".equals(rawType)) {
+ return "Integer";
+ } else if (rawType.toLowerCase().equals(rawType)) {
+ return ConfiggenUtil.capitalize(rawType);
+ } else {
+ return rawType;
+ }
+ }
+}
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java b/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java
index 299a5540098..995ef419f30 100644
--- a/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java
+++ b/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java
@@ -26,7 +26,7 @@ public class ConfiggenUtil {
return className;
}
- private static String capitalize(String in) {
+ static String capitalize(String in) {
StringBuilder sb = new StringBuilder(in);
sb.setCharAt(0, Character.toTitleCase(in.charAt(0)));
return sb.toString();
diff --git a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java
new file mode 100644
index 00000000000..00498094db5
--- /dev/null
+++ b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java
@@ -0,0 +1,170 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.yahoo.config.codegen.ConfigGenerator.indentCode;
+import static com.yahoo.config.codegen.ConfiggenUtil.createClassName;
+import static com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX;
+
+/**
+ * Builds one Java class based on the given CNode tree.
+ *
+ * @author gjoranv
+ * @author Tony Vaagenes
+ * @author ollivir
+ */
+public class JavaClassBuilder implements ClassBuilder {
+ public static final String INDENTATION = " ";
+
+ private final InnerCNode root;
+ private final NormalizedDefinition nd;
+ private final String packagePrefix;
+ private final String javaPackage;
+ private final String className;
+ private final File destDir;
+
+ public JavaClassBuilder(InnerCNode root, NormalizedDefinition nd, File destDir, String rawPackagePrefix) {
+ this.root = root;
+ this.nd = nd;
+ this.packagePrefix = (rawPackagePrefix != null) ? rawPackagePrefix : DEFAULT_PACKAGE_PREFIX;
+ this.javaPackage = (root.getPackage() != null) ? root.getPackage() : packagePrefix + root.getNamespace();
+ this.className = createClassName(root.getName());
+ this.destDir = destDir;
+ }
+
+ @Override
+ public void createConfigClasses() {
+ try {
+ File outFile = new File(getDestPath(destDir, javaPackage), className + ".java");
+ try (PrintStream out = new PrintStream(new FileOutputStream(outFile))) {
+ out.print(getConfigClass(className));
+ }
+ System.err.println(outFile.getPath() + " successfully written.");
+ } catch (FileNotFoundException e) {
+ throw new CodegenRuntimeException(e);
+ }
+ }
+
+ public String getConfigClass(String className) {
+ return getHeader() + "\n\n" + //
+ getRootClassDeclaration(root, className) + "\n\n" + //
+ indentCode(INDENTATION, getFrameworkCode()) + "\n\n" + //
+ ConfigGenerator.generateContent(INDENTATION, root, true) + "\n" + //
+ "}\n";
+ }
+
+ private String getHeader() {
+ return "/**\n" + //
+ " * This file is generated from a config definition file.\n" + //
+ " * ------------ D O N O T E D I T ! ------------\n" + //
+ " */\n" + //
+ "\n" + //
+ "package " + javaPackage + ";\n" + //
+ "\n" + //
+ "import java.util.*;\n" + //
+ "import java.nio.file.Path;\n" + //
+ "import edu.umd.cs.findbugs.annotations.NonNull;\n" + //
+ getImportFrameworkClasses(root.getNamespace());
+ }
+
+ private String getImportFrameworkClasses(String namespace) {
+ if (CNode.DEFAULT_NAMESPACE.equals(namespace) == false) {
+ return "import " + packagePrefix + CNode.DEFAULT_NAMESPACE + ".*;";
+ } else {
+ return "";
+ }
+ }
+
+ // TODO: remove the extra comment line " *" if root.getCommentBlock is empty
+ private String getRootClassDeclaration(InnerCNode root, String className) {
+ return "/**\n" + //
+ " * This class represents the root node of " + root.getFullName() + "\n" + //
+ " *\n" + //
+ "" + root.getCommentBlock(" *") + " */\n" + //
+ "public final class " + className + " extends ConfigInstance {\n" + //
+ "\n" + //
+ " public final static String CONFIG_DEF_MD5 = \"" + root.getMd5() + "\";\n" + //
+ " public final static String CONFIG_DEF_NAME = \"" + root.getName() + "\";\n" + //
+ " public final static String CONFIG_DEF_NAMESPACE = \"" + root.getNamespace() + "\";\n" + //
+ " public final static String CONFIG_DEF_VERSION = \"" + root.getVersion() + "\";\n" + //
+ " public final static String[] CONFIG_DEF_SCHEMA = {\n" + //
+ "" + indentCode(INDENTATION + INDENTATION, getDefSchema()) + "\n" + //
+ " };\n" + //
+ "\n" + //
+ " public static String getDefMd5() { return CONFIG_DEF_MD5; }\n" + //
+ " public static String getDefName() { return CONFIG_DEF_NAME; }\n" + //
+ " public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }\n" + //
+ " public static String getDefVersion() { return CONFIG_DEF_VERSION; }";
+ }
+
+ private String getDefSchema() {
+ return nd.getNormalizedContent().stream().map(l -> "\"" + l.replace("\"", "\\\"") + "\"").collect(Collectors.joining(",\n"));
+ }
+
+ private String getFrameworkCode() {
+ return "public interface Producer extends ConfigInstance.Producer {\n" + //
+ " void getConfig(Builder builder);\n" + //
+ "}";
+ }
+
+ /**
+ * @param rootDir
+ * The root directory for the destination path.
+ * @param javaPackage
+ * The java package
+ * @return the destination path for the generated config file, including the
+ * given rootDir.
+ */
+ private File getDestPath(File rootDir, String javaPackage) {
+ File dir = rootDir;
+ for (String subDir : javaPackage.split("\\.")) {
+ dir = new File(dir, subDir);
+ synchronized (this) {
+ if (!dir.isDirectory() && !dir.mkdir()) {
+ throw new CodegenRuntimeException("Could not create " + dir.getPath());
+ }
+ }
+ }
+ return dir;
+ }
+
+ /**
+ * Returns a name that can be safely used as a local variable in the generated
+ * config class for the given node. The name will be based on the given basis
+ * string, but the basis itself is not a possible return value.
+ *
+ * @param node
+ * The node to find a unused symbol name for.
+ * @param basis
+ * The basis for the generated symbol name.
+ * @return A name that is not used in the given config node.
+ */
+ static String createUniqueSymbol(CNode node, String basis) {
+ Set<String> usedSymbols = Arrays.stream(node.getChildren()).map(CNode::getName).collect(Collectors.toSet());
+ Random rng = new Random();
+
+ for (int i = 1;; i++) {
+ String candidate = (i < basis.length()) ? basis.substring(0, i)
+ : ReservedWords.INTERNAL_PREFIX + basis + rng.nextInt(Integer.MAX_VALUE);
+ if (usedSymbols.contains(candidate) == false) {
+ return candidate;
+ }
+ }
+ }
+
+ public String className() {
+ return className;
+ }
+
+ public String javaPackage() {
+ return javaPackage;
+ }
+}
diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala
deleted file mode 100644
index 4f6f310e32e..00000000000
--- a/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala
+++ /dev/null
@@ -1,350 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.config.codegen
-
-import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix}
-import JavaClassBuilder.{Indentation, createUniqueSymbol}
-import ConfigGenerator.{indentCode, nodeClass, userDataType, boxedDataType}
-import com.yahoo.config.codegen.LeafCNode._
-
-/**
- * @author gjoranv
- */
-
-object BuilderGenerator {
-
- def getBuilder(node: InnerCNode): String = {
- getDeclaration(node) + "\n" +
- indentCode(Indentation,
- getUninitializedScalars(node) + "\n\n" +
- node.getChildren.map(getBuilderFieldDefinition).mkString("\n") + "\n\n" +
- getBuilderConstructors(node, nodeClass(node)) + "\n\n" +
- getOverrideMethod(node) + "\n\n" +
- getBuilderSetters(node) + "\n" +
- getSpecialRootBuilderCode(node)
- ) +
- "}"
- }
-
- private def getDeclaration(node: InnerCNode) = {
- def getInterfaces =
- if (node.getParent == null) "implements ConfigInstance.Builder"
- else "implements ConfigBuilder"
-
- "public static class Builder " + getInterfaces + " {"
- }
-
- private def getSpecialRootBuilderCode(node: InnerCNode) = {
- if (node.getParent == null) "\n" + getDispatchCode(node) + "\n"
- else ""
- }
-
- private def getDispatchCode(node: InnerCNode) = {
- // Use full path to @Override, as users are free to define an inner node called 'override'. (summarymap.def does)
- // The generated inner 'Override' class would otherwise be mistaken for the annotation.
- """
- |@java.lang.Override
- |public final boolean dispatchGetConfig(ConfigInstance.Producer producer) {
- | if (producer instanceof Producer) {
- | ((Producer)producer).getConfig(this);
- | return true;
- | }
- | return false;
- |}
- |
- |@java.lang.Override
- |public final String getDefMd5() { return CONFIG_DEF_MD5; }
- |@java.lang.Override
- |public final String getDefName() { return CONFIG_DEF_NAME; }
- |@java.lang.Override
- |public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }
- """.stripMargin.trim
- }
-
- private def getUninitializedScalars(node: InnerCNode): String = {
- val scalarsWithoutDefault = {
- node.getChildren.collect {
- case leaf: LeafCNode if (!leaf.isArray && !leaf.isMap && leaf.getDefaultValue == null) =>
- "\"" + leaf.getName + "\""
- }
- }
-
- val uninitializedList =
- if (scalarsWithoutDefault.size > 0)
- "Arrays.asList(\n" + indentCode(Indentation, scalarsWithoutDefault.mkString("",",\n","\n)"))
- else
- ""
-
- "private Set<String> " + InternalPrefix + "uninitialized = new HashSet<String>(" + uninitializedList + ");"
- }
-
- private def getBuilderFieldDefinition(node: CNode): String = {
-
- (node match {
- case array if node.isArray =>
- "public List<%s> %s = new ArrayList<>()".format(builderType(array), array.getName)
- case map if node.isMap =>
- "public Map<String, %s> %s = new LinkedHashMap<>()".format(builderType(map), map.getName)
- case struct: InnerCNode =>
- "public %s %s = new %s()".format(builderType(struct), struct.getName, builderType(struct))
- case scalar : LeafCNode =>
- "private " + boxedBuilderType(scalar) + " " + scalar.getName + " = null"
- }) + ";"
- }
-
- private def getBuilderSetters(node: CNode): String = {
- val children: Array[CNode] = node.getChildren
-
- def structSetter(node: InnerCNode) = {
- <code>
- |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{
- | {node.getName} = {InternalPrefix}builder;
- | return this;
- |}}
- </code>.text.stripMargin.trim
- }
-
- def innerArraySetters(node: InnerCNode) = {
- <code>
- |/**
- | * Add the given builder to this builder's list of {nodeClass(node)} builders
- | * @param {InternalPrefix}builder a builder
- | * @return this builder
- | */
- |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{
- | {node.getName}.add({InternalPrefix}builder);
- | return this;
- |}}
- |
- |/**
- | * Set the given list as this builder's list of {nodeClass(node)} builders
- | * @param __builders a list of builders
- | * @return this builder
- | */
- |public Builder {node.getName}(List&lt;{builderType(node)}&gt; __builders) {{
- | {node.getName} = __builders;
- | return this;
- |}}
- </code>.text.stripMargin.trim
- }
-
- def leafArraySetters(node: LeafCNode) = {
- val setters =
- <code>
- |public Builder {node.getName}({builderType(node)} {InternalPrefix}value) {{
- | {node.getName}.add({InternalPrefix}value);
- | return this;
- |}}
- |
- |public Builder {node.getName}(Collection&lt;{builderType(node)}&gt; {InternalPrefix}values) {{
- | {node.getName}.addAll({InternalPrefix}values);
- | return this;
- |}}
- </code>.text.stripMargin.trim
-
- val privateSetter =
- if (builderType(node) == "String" || builderType(node) == "FileReference")
- ""
- else
- "\n\n" +
- <code>
- |
- |
- |private Builder {node.getName}(String {InternalPrefix}value) {{
- | return {node.getName}({builderType(node)}.valueOf({InternalPrefix}value));
- |}}
- </code>.text.stripMargin.trim
-
- setters + privateSetter
- }
-
- def innerMapSetters(node: CNode) = {
- <code>
- |public Builder {node.getName}(String {InternalPrefix}key, {builderType(node)} {InternalPrefix}value) {{
- | {node.getName}.put({InternalPrefix}key, {InternalPrefix}value);
- | return this;
- |}}
- |
- |public Builder {node.getName}(Map&lt;String, {builderType(node)}&gt; {InternalPrefix}values) {{
- | {node.getName}.putAll({InternalPrefix}values);
- | return this;
- |}}
- </code>.text.stripMargin.trim
- }
-
- def leafMapSetters(node: LeafCNode) = {
- val privateSetter =
- if (builderType(node) == "String" || builderType(node) == "FileReference")
- ""
- else
- "\n\n" +
- <code>
- |
- |
- |private Builder {node.getName}(String {InternalPrefix}key, String {InternalPrefix}value) {{
- | return {node.getName}({InternalPrefix}key, {builderType(node)}.valueOf({InternalPrefix}value));
- |}}
- </code>.text.stripMargin.trim
-
- innerMapSetters(node) + privateSetter
- }
-
- def scalarSetters(node: LeafCNode): String = {
- val name = node.getName
-
- val signalInitialized =
- if (node.getDefaultValue == null) InternalPrefix + "uninitialized.remove(\"" + name + "\");\n"
- else ""
-
- val stringSetter =
- builderType(node) match {
- case "String" => ""
- case "FileReference" => ""
- case _ =>
- """|
- |private Builder %s(String %svalue) {
- | return %s(%s.valueOf(%svalue));
- |}""".stripMargin.format(name, InternalPrefix,
- name, boxedDataType(node), InternalPrefix)
- }
-
- def getNullGuard = {
- if (builderType(node) != boxedBuilderType(node))
- ""
- else
- "\n" + "if (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");"
- .format(InternalPrefix)
- }
-
- // TODO: check if 2.9.2 allows string to start with a newline
- """|public Builder %s(%s %svalue) {%s
- | %s = %svalue;
- | %s
- """.stripMargin.format(name, builderType(node), InternalPrefix, getNullGuard,
- name, InternalPrefix,
- signalInitialized).trim +
- "\n return this;" + "\n}\n" +
- stringSetter
- }
-
- (children collect {
- case innerArray: InnerCNode if innerArray.isArray => innerArraySetters(innerArray)
- case innerMap: InnerCNode if innerMap.isMap => innerMapSetters(innerMap)
- case leafArray: LeafCNode if leafArray.isArray => leafArraySetters(leafArray)
- case leafMap: LeafCNode if leafMap.isMap => leafMapSetters(leafMap)
- case struct: InnerCNode => structSetter(struct)
- case scalar: LeafCNode => scalarSetters(scalar)
- } ).mkString("\n\n")
- }
-
- private def getBuilderConstructors(node: CNode, className: String): String = {
- def setBuilderValueFromConfig(child: CNode) = {
- val name = child.getName
- val isArray = child.isArray
- val isMap = child.isMap
-
- child match {
- case fileArray: FileLeaf if isArray => name + "(" + userDataType(fileArray) + ".toValues(config." + name + "()));"
- case fileMap: FileLeaf if isMap => name + "(" + userDataType(fileMap) + ".toValueMap(config." + name + "()));"
- case file: FileLeaf => name + "(config." + name + "().value());"
- case pathArray: PathLeaf if isArray => name + "(" + nodeClass(pathArray) + ".toFileReferences(config." + name + "));"
- case pathMap: PathLeaf if isMap => name + "(" + nodeClass(pathMap) + ".toFileReferenceMap(config." + name + "));"
- case path: PathLeaf => name + "(config." + name + ".getFileReference());"
- case leaf: LeafCNode => name + "(config." + name + "());"
- case innerArray: InnerCNode if isArray => setInnerArrayBuildersFromConfig(innerArray)
- case innerMap: InnerCNode if isMap => setInnerMapBuildersFromConfig(innerMap)
- case struct => name + "(new " + builderType(struct) + "(config." + name + "()));"
- }
- }
-
- def setInnerArrayBuildersFromConfig(innerArr: InnerCNode) = {
- val elemName = createUniqueSymbol(node, innerArr.getName)
- <code>
- |for ({userDataType(innerArr)} {elemName} : config.{innerArr.getName}()) {{
- | {innerArr.getName}(new {builderType(innerArr)}({elemName}));
- |}}
- </code>.text.stripMargin.trim
- }
-
- def setInnerMapBuildersFromConfig(innerMap: InnerCNode) = {
- val entryName = InternalPrefix + "entry"
- <code>
- |for (Map.Entry&lt;String, {userDataType(innerMap)}&gt; {entryName} : config.{innerMap.getName}().entrySet()) {{
- | {innerMap.getName}({entryName}.getKey(), new {userDataType(innerMap)}.Builder({entryName}.getValue()));
- |}}
- </code>.text.stripMargin.trim
- }
-
- <code>
- |public Builder() {{ }}
- |
- |public Builder({className} config) {{
- |{indentCode(Indentation, node.getChildren.map(setBuilderValueFromConfig).mkString("\n"))}
- |}}
- </code>.text.stripMargin.trim
- }
-
- def arrayOverride(name: String, superior: String): String = {
- Indentation + name + ".addAll(" + superior + "." + name + ");"
- }
-
- private def getOverrideMethod(node:CNode): String = {
- val method = "override"
- val superior = InternalPrefix + "superior"
-
- def callSetter(name: String): String = {
- name + "(" + superior + "." + name + ");"
- }
- def overrideBuilderValue(child: CNode) = {
- val name = child.getName
- child match {
- case leafArray: CNode if (child.isArray) =>
- conditionStatement(child) + "\n" + arrayOverride(name, superior)
- case struct: InnerCNode if !(child.isArray || child.isMap) =>
- name + "(" + name + "." + method + "(" + superior + "." + name + "));"
- case map: CNode if child.isMap =>
- callSetter(name)
- case _ =>
- conditionStatement(child) + "\n" +
- Indentation + callSetter(name)
- }
- }
-
- def conditionStatement(child: CNode) = {
- val name = child.getName
- val isArray = child.isArray
- val isMap = child.isMap
- child match {
- case _ if isArray => "if (!" + superior + "." + name + ".isEmpty())"
- case _ if isMap => ""
- case scalar: LeafCNode => "if (" + superior + "." + name + " != null)"
- case struct => ""
- }
- }
-
- <code>
- |private Builder {method}(Builder {superior}) {{
- |{indentCode(Indentation, node.getChildren.map(overrideBuilderValue).mkString("\n"))}
- | return this;
- |}}
- </code>.text.stripMargin.trim
- }
-
- private def builderType(node: CNode): String = {
- node match {
- case inner: InnerCNode => boxedDataType(node) + ".Builder"
- case file: FileLeaf => "String"
- case path: PathLeaf => "FileReference"
- case leafArray: LeafCNode if (node.isArray || node.isMap) => boxedDataType(node)
- case _ => userDataType(node)
- }
- }
-
- private def boxedBuilderType(node: LeafCNode): String = {
- node match {
- case file: FileLeaf => "String"
- case path: PathLeaf => "FileReference"
- case _ => boxedDataType(node)
- }
- }
-
-}
diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala
deleted file mode 100644
index 38306a03575..00000000000
--- a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala
+++ /dev/null
@@ -1,423 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.config.codegen
-
-
-import com.yahoo.config.codegen.BuilderGenerator.getBuilder
-import com.yahoo.config.codegen.JavaClassBuilder.Indentation
-import com.yahoo.config.codegen.LeafCNode._
-import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix}
-
-/**
- * @author gjoranv
- * @author tonytv
- */
-// TODO: don't take indent as method param - the caller should indent
-object ConfigGenerator {
-
- def generateContent(indent: String, node: InnerCNode, isOuter: Boolean = true): String = {
- val children: Array[CNode] = node.getChildren
-
- def generateCodeForChildren: String = {
- (children collect {
- case enum: EnumLeaf => getEnumCode(enum, "") + "\n"
- case inner: InnerCNode => getInnerDefinition(inner, indent) + "\n"
- } ).mkString("\n")
- }
-
- def getInnerDefinition(inner: InnerCNode, indent: String) = {
- <code>
- |{getClassDoc(inner, indent)}
- |{getClassDeclaration(inner)}
- |{generateContent(indent, inner, false)}
- </code>.text.stripMargin.trim + "\n}"
- }
-
- def getClassDeclaration(node: CNode): String = {
- "public final static class " + nodeClass(node)+ " extends InnerNode { " + "\n"
- }
-
- def getFieldDefinition(node: CNode): String = {
- node.getCommentBlock("//") + "private final " +
- (node match {
- case _: LeafCNode if node.isArray =>
- "LeafNodeVector<%s, %s> %s;".format(boxedDataType(node), nodeClass(node), node.getName)
- case _: InnerCNode if node.isArray =>
- "InnerNodeVector<%s> %s;".format(nodeClass(node), node.getName)
- case _ if node.isMap =>
- "Map<String, %s> %s;".format(nodeClass(node), node.getName)
- case _ =>
- "%s %s;".format(nodeClass(node), node.getName)
- })
- }
-
- def getStaticMethods = {
- if (node.isArray) getStaticMethodsForInnerArray(node) + "\n\n"
- else if (node.isMap) getStaticMethodsForInnerMap(node) + "\n\n"
- else ""
- }
-
- def getContainsFieldsFlaggedWithRestart(node: CNode): String = {
- if (isOuter) {
- """
- |private static boolean containsFieldsFlaggedWithRestart() {
- | return %b;
- |}
- """.stripMargin.trim.format(node.needRestart) + "\n\n"
- } else ""
- }
-
- indentCode(indent,
- getBuilder(node) + "\n\n" +
- children.map(getFieldDefinition).mkString("\n") + "\n\n" +
- getConstructors(node) + "\n\n" +
- getAccessors(children) + "\n\n" +
- getGetChangesRequiringRestart(node) + "\n\n" +
- getContainsFieldsFlaggedWithRestart(node) +
- getStaticMethods +
- generateCodeForChildren
- )
- }
-
- private def getGetChangesRequiringRestart(node: InnerCNode): String = {
- def quotedComment(node: CNode): String = {
- node.getComment.replace("\n", "\\n").replace("\"", "\\\"")
- }
-
- def getComparison(node: CNode): String = node match {
- case inner: InnerCNode if inner.isArray =>
- <code>
- | changes.compareArray(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}",
- | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b));
- </code>.text.stripMargin.trim
- case inner: InnerCNode if inner.isMap =>
- <code>
- | changes.compareMap(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}",
- | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b));
- </code>.text.stripMargin.trim
- case inner: InnerCNode =>
- <code>
- | changes.mergeChanges("{inner.getName}", this.{inner.getName}.getChangesRequiringRestart(newConfig.{inner.getName}));
- </code>.text.stripMargin.trim
- case node: CNode if node.isArray =>
- <code>
- | changes.compareArray(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}",
- | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}"));
- </code>.text.stripMargin.trim
- case node: CNode if node.isMap =>
- <code>
- | changes.compareMap(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}",
- | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}"));
- </code>.text.stripMargin.trim
- case node: CNode =>
- <code>
- | changes.compare(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}");
- </code>.text.stripMargin.trim
- }
-
- val comparisons =
- for {
- c <- node.getChildren if c.needRestart
- } yield "\n " + getComparison(c)
-
- <code>
- |private ChangesRequiringRestart getChangesRequiringRestart({nodeClass(node)} newConfig) {{
- | ChangesRequiringRestart changes = new ChangesRequiringRestart("{node.getName}");{comparisons.mkString("")}
- | return changes;
- |}}
- </code>.text.stripMargin.trim
- }
-
-
- private def scalarDefault(scalar: LeafCNode): String = {
- scalar match {
- case _ if scalar.getDefaultValue == null => ""
- case enumWithNullDefault: EnumLeaf if enumWithNullDefault.getDefaultValue.getValue == null => ""
- case enum: EnumLeaf => nodeClass(enum) + "." + enum.getDefaultValue.getStringRepresentation
- case long: LongLeaf => long.getDefaultValue.getStringRepresentation + "L"
- case double: DoubleLeaf => double.getDefaultValue.getStringRepresentation + "D"
- case _ => scalar.getDefaultValue.getStringRepresentation
- }
- }
-
- private def getConstructors(inner: InnerCNode) = {
-
- def assignFromBuilder(child: CNode) = {
- val name = child.getName
- val className = nodeClass(child)
- val dataType = boxedDataType(child)
- val isArray = child.isArray
- val isMap = child.isMap
-
- def assignIfInitialized(leaf: LeafCNode) = {
- <code>
- |{name} = (builder.{name} == null) ?
- | new {className}({scalarDefault(leaf)}) : new {className}(builder.{name});
- </code>.text.stripMargin.trim
- }
-
- child match {
- case fileArray: FileLeaf if isArray =>
- name + " = LeafNodeVector.createFileNodeVector(builder."+ name +");"
- case pathArray: PathLeaf if isArray =>
- name + " = LeafNodeVector.createPathNodeVector(builder."+ name +");"
- case leafArray: LeafCNode if isArray =>
- name + " = new LeafNodeVector<>(builder."+ name +", new " + className + "());"
- case fileMap: LeafCNode if isMap && child.isInstanceOf[FileLeaf] =>
- name + " = LeafNodeMaps.asFileNodeMap(builder."+ name +");"
- case pathMap: LeafCNode if isMap && child.isInstanceOf[PathLeaf] =>
- name + " = LeafNodeMaps.asPathNodeMap(builder."+ name +");"
- case leafMap: LeafCNode if isMap =>
- name + " = LeafNodeMaps.asNodeMap(builder."+ name +", new " + className + "());"
- case innerArray: InnerCNode if isArray =>
- name + " = " + className + ".createVector(builder." + name + ");"
- case innerMap: InnerCNode if isMap =>
- name + " = " + className + ".createMap(builder." + name + ");"
- case struct: InnerCNode =>
- name + " = new " + className + "(builder." + name + ", throwIfUninitialized);"
- case leaf: LeafCNode =>
- assignIfInitialized(leaf)
- }
- }
-
- // TODO: merge these two constructors into one when the config library uses builders to set values from payload.
- <code>
- |public {nodeClass(inner)}(Builder builder) {{
- | this(builder, true);
- |}}
- |
- |private {nodeClass(inner)}(Builder builder, boolean throwIfUninitialized) {{
- | if (throwIfUninitialized &amp;&amp; ! builder.{InternalPrefix}uninitialized.isEmpty())
- | throw new IllegalArgumentException("The following builder parameters for " +
- | "{inner.getFullName} must be initialized: " + builder.{InternalPrefix}uninitialized);
- |
- |{indentCode(Indentation, inner.getChildren.map(assignFromBuilder).mkString("\n"))}
- |}}
- </code>.text.stripMargin.trim
- }
-
- private def getAccessors(children: Array[CNode]): String = {
-
- def getAccessorCode(indent: String, node: CNode): String = {
- indentCode(indent,
- if (node.isArray)
- accessorsForArray(node)
- else if (node.isMap)
- accessorsForMap(node)
- else
- accessorForStructOrScalar(node))
- }
-
- def valueAccessor(node: CNode) = node match {
- case leaf: LeafCNode => ".value()"
- case inner => ""
- }
-
- def listAccessor(node: CNode) = node match {
- case leaf: LeafCNode => "%s.asList()".format(leaf.getName)
- case inner => inner.getName
- }
-
- def mapAccessor(node: CNode) = node match {
- case leaf: LeafCNode => "LeafNodeMaps.asValueMap(%s)".format(leaf.getName)
- case inner => "Collections.unmodifiableMap(%s)".format(inner.getName)
- }
-
- def accessorsForArray(node: CNode): String = {
- val name = node.getName
- val fullName = node.getFullName
- <code>
- |/**
- | * @return {fullName}
- | */
- |public List&lt;{boxedDataType(node)}&gt; {name}() {{
- | return {listAccessor(node)};
- |}}
- |
- |/**
- | * @param i the index of the value to return
- | * @return {fullName}
- | */
- |public {userDataType(node)} {name}(int i) {{
- | return {name}.get(i){valueAccessor(node)};
- |}}
- </code>.text.stripMargin.trim
- }
-
- def accessorsForMap(node: CNode): String = {
- val name = node.getName
- val fullName = node.getFullName
- <code>
- |/**
- | * @return {fullName}
- | */
- |public Map&lt;String, {boxedDataType(node)}&gt; {name}() {{
- | return {mapAccessor(node)};
- |}}
- |
- |/**
- | * @param key the key of the value to return
- | * @return {fullName}
- | */
- |public {userDataType(node)} {name}(String key) {{
- | return {name}.get(key){valueAccessor(node)};
- |}}
- </code>.text.stripMargin.trim
- }
-
- def accessorForStructOrScalar(node: CNode): String = {
- <code>
- |/**
- | * @return {node.getFullName}
- | */
- |public {userDataType(node)} {node.getName}() {{
- | return {node.getName}{valueAccessor(node)};
- |}}
- </code>.text.stripMargin.trim
- }
-
- val accessors =
- for {
- c <- children
- accessor = getAccessorCode("", c)
- if (accessor.length > 0)
- } yield (accessor + "\n")
- accessors.mkString("\n").trim
- }
-
- private def getStaticMethodsForInnerArray(inner: InnerCNode) = {
- """
- |private static InnerNodeVector<%s> createVector(List<Builder> builders) {
- | List<%s> elems = new ArrayList<>();
- | for (Builder b : builders) {
- | elems.add(new %s(b));
- | }
- | return new InnerNodeVector<%s>(elems);
- |}
- """.stripMargin.format(List.fill(5)(nodeClass(inner)): _*).trim
- }
-
- private def getStaticMethodsForInnerMap(inner: InnerCNode) = {
- """
- |private static Map<String, %s> createMap(Map<String, Builder> builders) {
- | Map<String, %s> ret = new LinkedHashMap<>();
- | for(String key : builders.keySet()) {
- | ret.put(key, new %s(builders.get(key)));
- | }
- | return Collections.unmodifiableMap(ret);
- |}
- """.stripMargin.format(List.fill(3)(nodeClass(inner)): _*).trim
- }
-
- private def getEnumCode(enum: EnumLeaf, indent: String): String = {
-
- def getEnumValues(enum: EnumLeaf): String = {
- val enumValues =
- for (value <- enum.getLegalValues) yield
- """ public final static Enum %s = Enum.%s;""".format(value, value)
- enumValues.mkString("\n")
- }
-
- // TODO: try to rewrite to xml
- val code =
- """
- |%s
- |public final static class %s extends EnumNode<%s> {
-
- | public %s(){
- | this.value = null;
- | }
-
- | public %s(Enum enumValue) {
- | super(enumValue != null);
- | this.value = enumValue;
- | }
-
- | public enum Enum {%s}
- |%s
-
- | @Override
- | protected boolean doSetValue(@NonNull String name) {
- | try {
- | value = Enum.valueOf(name);
- | return true;
- | } catch (IllegalArgumentException e) {
- | }
- | return false;
- | }
- |}
- |"""
- .stripMargin.format(getClassDoc(enum, indent),
- nodeClass(enum),
- nodeClass(enum)+".Enum",
- nodeClass(enum),
- nodeClass(enum),
- enum.getLegalValues.mkString(", "),
- getEnumValues(enum))
-
- indentCode(indent, code).trim
- }
-
- def getClassDoc(node: CNode, indent: String): String = {
- val header = "/**\n" + " * This class represents " + node.getFullName
- val nodeComment = node.getCommentBlock(" *") match {
- case "" => ""
- case s => "\n *\n" + s.stripLineEnd // TODO: strip trailing \n in CNode.getCommentBlock
- }
- header + nodeComment + "\n */"
- }
-
- def indentCode(indent: String, code: String): String = {
- val indentedLines =
- for (s <- code.split("\n", -1)) yield
- if (s.length() > 0) (indent + s) else s
- indentedLines.mkString("\n")
- }
-
- /**
- * @return the name of the class that is generated by this node.
- */
- def nodeClass(node: CNode): String = {
- node match {
- case emptyName: CNode if node.getName.length == 0 =>
- throw new CodegenRuntimeException("Node with empty name, under parent " + emptyName.getParent.getName)
- case root: InnerCNode if root.getParent == null => ConfiggenUtil.createClassName(root.getName)
- case b: BooleanLeaf => "BooleanNode"
- case d: DoubleLeaf => "DoubleNode"
- case f: FileLeaf => "FileNode"
- case p: PathLeaf => "PathNode"
- case i: IntegerLeaf => "IntegerNode"
- case l: LongLeaf => "LongNode"
- case r: ReferenceLeaf => "ReferenceNode"
- case s: StringLeaf => "StringNode"
- case _ => node.getName.capitalize
- }
- }
-
- def userDataType(node: CNode): String = {
- node match {
- case inner: InnerCNode => nodeClass(node)
- case enum: EnumLeaf => nodeClass(enum) + ".Enum"
- case b: BooleanLeaf => "boolean"
- case d: DoubleLeaf => "double"
- case f: FileLeaf => "FileReference"
- case p: PathLeaf => "Path"
- case i: IntegerLeaf => "int"
- case l: LongLeaf => "long"
- case s: StringLeaf => "String"
- }
- }
-
- /**
- * @return the boxed java data type, e.g. Integer for int
- */
- def boxedDataType(node: CNode): String = {
- val rawType = userDataType(node)
-
- rawType match {
- case "int" => "Integer"
- case _ if rawType == rawType.toLowerCase => rawType.capitalize
- case _ => rawType
- }
- }
-
-}
diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala b/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala
deleted file mode 100644
index e03a6d3d04b..00000000000
--- a/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala
+++ /dev/null
@@ -1,186 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.config.codegen
-
-import java.io.{File, FileNotFoundException, FileOutputStream, PrintStream}
-
-import com.yahoo.config.codegen.ConfigGenerator.indentCode
-import com.yahoo.config.codegen.ConfiggenUtil.createClassName
-import com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX
-
-import scala.collection.JavaConverters._
-import scala.util.Random
-/**
- * Builds one Java class based on the given CNode tree.
- *
- * @author gjoranv
- * @author tonytv
- */
-class JavaClassBuilder(
- root: InnerCNode,
- nd: NormalizedDefinition,
- destDir: File,
- rawPackagePrefix: String)
- extends ClassBuilder
-{
- import JavaClassBuilder._
-
- val packagePrefix = if (rawPackagePrefix != null) rawPackagePrefix else DEFAULT_PACKAGE_PREFIX
- val javaPackage = if (root.getPackage != null) root.getPackage else packagePrefix + root.getNamespace
- val className = createClassName(root.getName)
-
- override def createConfigClasses() {
- try {
- val outFile = new File(getDestPath(destDir, javaPackage), className + ".java")
- var out: PrintStream = null
- try {
- out = new PrintStream(new FileOutputStream(outFile))
- out.print(getConfigClass(className))
- } finally {
- if (out != null) out.close()
- }
- System.err.println(outFile.getPath + " successfully written.")
- }
- catch {
- case e: FileNotFoundException => {
- throw new CodegenRuntimeException(e)
- }
- }
- }
-
- def getConfigClass(className:String): String = {
- val ret = new StringBuilder
-
- ret.append(getHeader).append("\n\n")
- ret.append(getRootClassDeclaration(root, className)).append("\n\n")
- ret.append(indentCode(Indentation, getFrameworkCode(className))).append("\n\n")
- ret.append(ConfigGenerator.generateContent(Indentation, root)).append("\n")
- ret.append("}\n")
-
- ret.toString()
- }
-
- private def getHeader: String = {
- <code>
- |/**
- | * This file is generated from a config definition file.
- | * ------------ D O N O T E D I T ! ------------
- | */
- |
- |package {javaPackage};
- |
- |import java.util.*;
- |import java.nio.file.Path;
- |import edu.umd.cs.findbugs.annotations.NonNull;
- |{getImportFrameworkClasses(root.getNamespace)}
- </code>.text.stripMargin.trim
- }
-
- private def getImportFrameworkClasses(namespace: String): String = {
- if (namespace != CNode.DEFAULT_NAMESPACE)
- "import " + packagePrefix + CNode.DEFAULT_NAMESPACE + ".*;\n"
- else
- ""
- }
-
- // TODO: remove the extra comment line " *" if root.getCommentBlock is empty
- private def getRootClassDeclaration(root:InnerCNode, className: String): String = {
- <code>
- |/**
- | * This class represents the root node of {root.getFullName}
- | *
- |{root.getCommentBlock(" *")} */
- |public final class {className} extends ConfigInstance {{
- |
- | public final static String CONFIG_DEF_MD5 = "{root.getMd5}";
- | public final static String CONFIG_DEF_NAME = "{root.getName}";
- | public final static String CONFIG_DEF_NAMESPACE = "{root.getNamespace}";
- | public final static String CONFIG_DEF_VERSION = "{root.getVersion}";
- | public final static String[] CONFIG_DEF_SCHEMA = {{
- |{indentCode(Indentation * 2, getDefSchema)}
- | }};
- |
- | public static String getDefMd5() {{ return CONFIG_DEF_MD5; }}
- | public static String getDefName() {{ return CONFIG_DEF_NAME; }}
- | public static String getDefNamespace() {{ return CONFIG_DEF_NAMESPACE; }}
- | public static String getDefVersion() {{ return CONFIG_DEF_VERSION; }}
- </code>.text.stripMargin.trim
- }
-
- private def getDefSchema: String = {
- nd.getNormalizedContent.asScala.map { line =>
- "\"" +
- line.replace("\"", "\\\"") +
- "\""
- }.mkString(",\n")
- }
-
- private def getFrameworkCode(className: String): String = {
- getProducerBase
- }
-
- private def getProducerBase = {
- """
- |public interface Producer extends ConfigInstance.Producer {
- | void getConfig(Builder builder);
- |}
- """.stripMargin.trim
- }
-
- /**
- * @param rootDir The root directory for the destination path.
- * @param javaPackage The java package
- * @return the destination path for the generated config file, including the given rootDir.
- */
- private def getDestPath(rootDir: File, javaPackage: String): File = {
- var dir: File = rootDir
- val subDirs: Array[String] = javaPackage.split("""\.""")
- for (subDir <- subDirs) {
- dir = new File(dir, subDir)
- this.synchronized {
- if (!dir.isDirectory && !dir.mkdir) throw new CodegenRuntimeException("Could not create " + dir.getPath)
- }
- }
- dir
- }
-
-}
-
-
-object JavaClassBuilder {
-
- val Indentation = " "
-
- /**
- * Returns a name that can be safely used as a local variable in the generated config class
- * for the given node. The name will be based on the given basis string, but the basis itself is
- * not a possible return value.
- *
- * @param node The node to find a unused symbol name for.
- * @param basis The basis for the generated symbol name.
- * @return A name that is not used in the given config node.
- */
- def createUniqueSymbol(node: CNode, basis: String) = {
-
- def getCandidate(cnt: Int) = {
- if (cnt < basis.length())
- basis.substring(0, cnt)
- else
- ReservedWords.INTERNAL_PREFIX + basis + Random.nextInt().abs
- }
-
- def getUsedSymbols: Set[String] = {
- (node.getChildren map (child => child.getName)).toSet
- }
-
- // TODO: refactoring potential
- val usedSymbols = getUsedSymbols
- var count = 1
- var candidate = getCandidate(count)
- while (usedSymbols contains(candidate)) {
- count += 1
- candidate = getCandidate(count)
- }
- candidate
- }
-
-}
diff --git a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java
new file mode 100644
index 00000000000..744f8c9b1a2
--- /dev/null
+++ b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java
@@ -0,0 +1,116 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.codegen;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+
+import static com.yahoo.config.codegen.ConfiggenUtil.createClassName;
+import static com.yahoo.config.codegen.JavaClassBuilder.createUniqueSymbol;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ * @author ollivir
+ */
+public class JavaClassBuilderTest {
+ private static final String TEST_DIR = "target/test-classes/";
+ private static final String DEF_NAME = TEST_DIR + "allfeatures.def";
+ private static final String REFERENCE_NAME = TEST_DIR + "allfeatures.reference";
+
+ @Ignore
+ @Test
+ public void visual_inspection_of_generated_class() {
+ final String testDefinition = "version=1\n" + //
+ "namespace=test\n" + //
+ "p path\n" + //
+ "pathArr[] path\n" + //
+ "f file\n" + //
+ "fileArr[] file\n" + //
+ "i int default=0\n" + //
+ "# A long value\n" + //
+ "l long default=0\n" + //
+ "s string default=\"\"\n" + //
+ "b bool\n" + //
+ "# An enum value\n" + //
+ "e enum {A, B, C}\n" + //
+ "intArr[] int\n" + //
+ "boolArr[] bool\n" + //
+ "enumArr[] enum {FOO, BAR}\n" + //
+ "intMap{} int\n" + //
+ "# A struct\n" + //
+ "# with multi-line\n" + //
+ "# comment and \"quotes\".\n" + //
+ "myStruct.i int\n" + //
+ "myStruct.s string\n" + //
+ "# An inner array\n" + //
+ "myArr[].i int\n" + //
+ "myArr[].newStruct.s string\n" + //
+ "myArr[].newStruct.b bool\n" + //
+ "myArr[].intArr[] int\n" + //
+ "# An inner map\n" + //
+ "myMap{}.i int\n" + //
+ "myMap{}.newStruct.s string\n" + //
+ "myMap{}.newStruct.b bool\n" + //
+ "myMap{}.intArr[] int\n" + //
+ "intMap{} int\n";
+
+ DefParser parser = new DefParser("test", new StringReader(testDefinition));
+ InnerCNode root = parser.getTree();
+ JavaClassBuilder builder = new JavaClassBuilder(root, parser.getNormalizedDefinition(), null, null);
+ String configClass = builder.getConfigClass("TestConfig");
+ System.out.print(configClass);
+ }
+
+ @Test
+ public void testCreateUniqueSymbol() {
+ final String testDefinition = "version=1\n" + //
+ "namespace=test\n" + //
+ "m int\n" + //
+ "n int\n";
+ InnerCNode root = new DefParser("test", new StringReader(testDefinition)).getTree();
+
+ assertThat(createUniqueSymbol(root, "foo"), is("f"));
+ assertThat(createUniqueSymbol(root, "name"), is("na"));
+ assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m"));
+
+ // The basis string is not a legal return value, even if unique, to avoid
+ // multiple symbols with the same name if the same basis string is given twice.
+ assertTrue(createUniqueSymbol(root, "my").startsWith(ReservedWords.INTERNAL_PREFIX + "my"));
+ }
+
+ @Test
+ public void testCreateClassName() {
+ assertThat(createClassName("simple"), is("SimpleConfig"));
+ assertThat(createClassName("a"), is("AConfig"));
+ assertThat(createClassName("a-b-c"), is("ABCConfig"));
+ assertThat(createClassName("a-1-2b"), is("A12bConfig"));
+ assertThat(createClassName("my-app"), is("MyAppConfig"));
+ assertThat(createClassName("MyApp"), is("MyAppConfig"));
+ }
+
+ @Test(expected = CodegenRuntimeException.class)
+ public void testIllegalClassName() {
+ createClassName("+illegal");
+ }
+
+ @Test
+ public void verify_generated_class_against_reference() throws IOException {
+ final String testDefinition = String.join("\n", Files.readAllLines(FileSystems.getDefault().getPath(DEF_NAME)));
+ final String referenceClass = String.join("\n", Files.readAllLines(FileSystems.getDefault().getPath(REFERENCE_NAME))) + "\n";
+
+ DefParser parser = new DefParser("allfeatures", new StringReader(testDefinition));
+ InnerCNode root = parser.getTree();
+ JavaClassBuilder builder = new JavaClassBuilder(root, parser.getNormalizedDefinition(), null, null);
+ String configClass = builder.getConfigClass("AllfeaturesConfig");
+
+ assertEquals(referenceClass, configClass);
+ }
+}
diff --git a/configgen/src/test/resources/allfeatures.reference b/configgen/src/test/resources/allfeatures.reference
new file mode 100644
index 00000000000..ebc21e8255c
--- /dev/null
+++ b/configgen/src/test/resources/allfeatures.reference
@@ -0,0 +1,1983 @@
+/**
+ * This file is generated from a config definition file.
+ * ------------ D O N O T E D I T ! ------------
+ */
+
+package com.yahoo.configgen;
+
+import java.util.*;
+import java.nio.file.Path;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import com.yahoo.config.*;
+
+/**
+ * This class represents the root node of allfeatures
+ *
+ * Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ *
+ * This def file should test most aspects of def files that makes a difference
+ * for the generated config classes. The goal is to trigger all blocks of
+ * code in the code generators. This includes:
+ *
+ * - Use all legal special characters in the def file name, to ensure that those
+ * that needs to be replaced in type names are actually replaced.
+ * - Use the same enum type twice to verify that we dont declare or define it
+ * twice.
+ * - Use the same struct type twice for the same reason.
+ * - Include arrays of primitives and structs.
+ * - Include enum primitives and array of enums. Arrays of enums must be handled
+ * specially by the C++ code.
+ * - Include enums both with and without default values.
+ * - Include primitive string, numbers & doubles both with and without default
+ * values.
+ * - Have an array within a struct, to verify that we correctly recurse.
+ * - Reuse type name further within to ensure that this works.
+ */
+public final class AllfeaturesConfig extends ConfigInstance {
+
+ public final static String CONFIG_DEF_MD5 = "eb2d24dbbcf054b21be729e2cfaafd93";
+ public final static String CONFIG_DEF_NAME = "allfeatures";
+ public final static String CONFIG_DEF_NAMESPACE = "configgen";
+ public final static String CONFIG_DEF_VERSION = "";
+ public final static String[] CONFIG_DEF_SCHEMA = {
+ "namespace=configgen",
+ "boolVal bool",
+ "bool_with_def bool default=false",
+ "intVal int",
+ "intWithDef int default=-545",
+ "longVal long",
+ "longWithDef long default=1234567890123",
+ "doubleVal double",
+ "double_with_def double default=-6.43",
+ "stringVal string",
+ "stringwithdef string default=\"foobar#notacomment\"",
+ "enumVal enum { FOO, BAR, FOOBAR }",
+ "enumwithdef enum { FOO2, BAR2, FOOBAR2 } default=BAR2",
+ "refVal reference",
+ "refwithdef reference default=\":parent:\"",
+ "fileVal file",
+ "pathVal path",
+ "boolarr[] bool",
+ "intarr[] int",
+ "longarr[] long",
+ "doublearr[] double",
+ "stringarr[] string",
+ "enumarr[] enum { ARRAY, VALUES }",
+ "refarr[] reference",
+ "filearr[] file",
+ "pathArr[] path",
+ "intMap{} int",
+ "pathMap{} file",
+ "basic_struct.foo string default=\"foo\"",
+ "basic_struct.bar int default=0",
+ "struct_of_struct.inner0.name string default=\"inner0\"",
+ "struct_of_struct.inner0.index int default=0",
+ "struct_of_struct.inner1.name string default=\"inner1\"",
+ "struct_of_struct.inner1.index int default=1",
+ "myArray[].intVal int default=14",
+ "myArray[].stringVal[] string",
+ "myArray[].enumVal enum { INNER, ENUM, TYPE } default=TYPE",
+ "myArray[].refVal reference",
+ "myArray[].anotherArray[].foo int default=-4",
+ "myMap{}.intVal int default=15",
+ "myMap{}.stringVal[] string",
+ "myMap{}.enumVal enum { INNER, ENUM, TYPE } default=ENUM",
+ "myMap{}.refVal reference",
+ "myMap{}.anotherArray[].foo int default=-5"
+ };
+
+ public static String getDefMd5() { return CONFIG_DEF_MD5; }
+ public static String getDefName() { return CONFIG_DEF_NAME; }
+ public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }
+ public static String getDefVersion() { return CONFIG_DEF_VERSION; }
+
+ public interface Producer extends ConfigInstance.Producer {
+ void getConfig(Builder builder);
+ }
+
+ public static class Builder implements ConfigInstance.Builder {
+ private Set<String> __uninitialized = new HashSet<String>(Arrays.asList(
+ "boolVal",
+ "intVal",
+ "longVal",
+ "doubleVal",
+ "stringVal",
+ "enumVal",
+ "refVal",
+ "fileVal",
+ "pathVal"
+ ));
+
+ private Boolean boolVal = null;
+ private Boolean bool_with_def = null;
+ private Integer intVal = null;
+ private Integer intWithDef = null;
+ private Long longVal = null;
+ private Long longWithDef = null;
+ private Double doubleVal = null;
+ private Double double_with_def = null;
+ private String stringVal = null;
+ private String stringwithdef = null;
+ private EnumVal.Enum enumVal = null;
+ private Enumwithdef.Enum enumwithdef = null;
+ private String refVal = null;
+ private String refwithdef = null;
+ private String fileVal = null;
+ private FileReference pathVal = null;
+ public List<Boolean> boolarr = new ArrayList<>();
+ public List<Integer> intarr = new ArrayList<>();
+ public List<Long> longarr = new ArrayList<>();
+ public List<Double> doublearr = new ArrayList<>();
+ public List<String> stringarr = new ArrayList<>();
+ public List<Enumarr.Enum> enumarr = new ArrayList<>();
+ public List<String> refarr = new ArrayList<>();
+ public List<String> filearr = new ArrayList<>();
+ public List<FileReference> pathArr = new ArrayList<>();
+ public Map<String, Integer> intMap = new LinkedHashMap<>();
+ public Map<String, String> pathMap = new LinkedHashMap<>();
+ public Basic_struct.Builder basic_struct = new Basic_struct.Builder();
+ public Struct_of_struct.Builder struct_of_struct = new Struct_of_struct.Builder();
+ public List<MyArray.Builder> myArray = new ArrayList<>();
+ public Map<String, MyMap.Builder> myMap = new LinkedHashMap<>();
+
+ public Builder() { }
+
+ public Builder(AllfeaturesConfig config) {
+ boolVal(config.boolVal());
+ bool_with_def(config.bool_with_def());
+ intVal(config.intVal());
+ intWithDef(config.intWithDef());
+ longVal(config.longVal());
+ longWithDef(config.longWithDef());
+ doubleVal(config.doubleVal());
+ double_with_def(config.double_with_def());
+ stringVal(config.stringVal());
+ stringwithdef(config.stringwithdef());
+ enumVal(config.enumVal());
+ enumwithdef(config.enumwithdef());
+ refVal(config.refVal());
+ refwithdef(config.refwithdef());
+ fileVal(config.fileVal().value());
+ pathVal(config.pathVal.getFileReference());
+ boolarr(config.boolarr());
+ intarr(config.intarr());
+ longarr(config.longarr());
+ doublearr(config.doublearr());
+ stringarr(config.stringarr());
+ enumarr(config.enumarr());
+ refarr(config.refarr());
+ filearr(FileReference.toValues(config.filearr()));
+ pathArr(PathNode.toFileReferences(config.pathArr));
+ intMap(config.intMap());
+ pathMap(FileReference.toValueMap(config.pathMap()));
+ basic_struct(new Basic_struct.Builder(config.basic_struct()));
+ struct_of_struct(new Struct_of_struct.Builder(config.struct_of_struct()));
+ for (MyArray m : config.myArray()) {
+ myArray(new MyArray.Builder(m));
+ }
+ for (Map.Entry<String, MyMap> __entry : config.myMap().entrySet()) {
+ myMap(__entry.getKey(), new MyMap.Builder(__entry.getValue()));
+ }
+ }
+
+ private Builder override(Builder __superior) {
+ if (__superior.boolVal != null)
+ boolVal(__superior.boolVal);
+ if (__superior.bool_with_def != null)
+ bool_with_def(__superior.bool_with_def);
+ if (__superior.intVal != null)
+ intVal(__superior.intVal);
+ if (__superior.intWithDef != null)
+ intWithDef(__superior.intWithDef);
+ if (__superior.longVal != null)
+ longVal(__superior.longVal);
+ if (__superior.longWithDef != null)
+ longWithDef(__superior.longWithDef);
+ if (__superior.doubleVal != null)
+ doubleVal(__superior.doubleVal);
+ if (__superior.double_with_def != null)
+ double_with_def(__superior.double_with_def);
+ if (__superior.stringVal != null)
+ stringVal(__superior.stringVal);
+ if (__superior.stringwithdef != null)
+ stringwithdef(__superior.stringwithdef);
+ if (__superior.enumVal != null)
+ enumVal(__superior.enumVal);
+ if (__superior.enumwithdef != null)
+ enumwithdef(__superior.enumwithdef);
+ if (__superior.refVal != null)
+ refVal(__superior.refVal);
+ if (__superior.refwithdef != null)
+ refwithdef(__superior.refwithdef);
+ if (__superior.fileVal != null)
+ fileVal(__superior.fileVal);
+ if (__superior.pathVal != null)
+ pathVal(__superior.pathVal);
+ if (!__superior.boolarr.isEmpty())
+ boolarr.addAll(__superior.boolarr);
+ if (!__superior.intarr.isEmpty())
+ intarr.addAll(__superior.intarr);
+ if (!__superior.longarr.isEmpty())
+ longarr.addAll(__superior.longarr);
+ if (!__superior.doublearr.isEmpty())
+ doublearr.addAll(__superior.doublearr);
+ if (!__superior.stringarr.isEmpty())
+ stringarr.addAll(__superior.stringarr);
+ if (!__superior.enumarr.isEmpty())
+ enumarr.addAll(__superior.enumarr);
+ if (!__superior.refarr.isEmpty())
+ refarr.addAll(__superior.refarr);
+ if (!__superior.filearr.isEmpty())
+ filearr.addAll(__superior.filearr);
+ if (!__superior.pathArr.isEmpty())
+ pathArr.addAll(__superior.pathArr);
+ intMap(__superior.intMap);
+ pathMap(__superior.pathMap);
+ basic_struct(basic_struct.override(__superior.basic_struct));
+ struct_of_struct(struct_of_struct.override(__superior.struct_of_struct));
+ if (!__superior.myArray.isEmpty())
+ myArray.addAll(__superior.myArray);
+ myMap(__superior.myMap);
+ return this;
+ }
+
+ public Builder boolVal(boolean __value) {
+ boolVal = __value;
+ __uninitialized.remove("boolVal");
+ return this;
+ }
+
+ private Builder boolVal(String __value) {
+ return boolVal(Boolean.valueOf(__value));
+ }
+
+ public Builder bool_with_def(boolean __value) {
+ bool_with_def = __value;
+ return this;
+ }
+
+ private Builder bool_with_def(String __value) {
+ return bool_with_def(Boolean.valueOf(__value));
+ }
+
+ public Builder intVal(int __value) {
+ intVal = __value;
+ __uninitialized.remove("intVal");
+ return this;
+ }
+
+ private Builder intVal(String __value) {
+ return intVal(Integer.valueOf(__value));
+ }
+
+ public Builder intWithDef(int __value) {
+ intWithDef = __value;
+ return this;
+ }
+
+ private Builder intWithDef(String __value) {
+ return intWithDef(Integer.valueOf(__value));
+ }
+
+ public Builder longVal(long __value) {
+ longVal = __value;
+ __uninitialized.remove("longVal");
+ return this;
+ }
+
+ private Builder longVal(String __value) {
+ return longVal(Long.valueOf(__value));
+ }
+
+ public Builder longWithDef(long __value) {
+ longWithDef = __value;
+ return this;
+ }
+
+ private Builder longWithDef(String __value) {
+ return longWithDef(Long.valueOf(__value));
+ }
+
+ public Builder doubleVal(double __value) {
+ doubleVal = __value;
+ __uninitialized.remove("doubleVal");
+ return this;
+ }
+
+ private Builder doubleVal(String __value) {
+ return doubleVal(Double.valueOf(__value));
+ }
+
+ public Builder double_with_def(double __value) {
+ double_with_def = __value;
+ return this;
+ }
+
+ private Builder double_with_def(String __value) {
+ return double_with_def(Double.valueOf(__value));
+ }
+
+ public Builder stringVal(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ stringVal = __value;
+ __uninitialized.remove("stringVal");
+ return this;
+ }
+
+
+ public Builder stringwithdef(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ stringwithdef = __value;
+ return this;
+ }
+
+
+ public Builder enumVal(EnumVal.Enum __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ enumVal = __value;
+ __uninitialized.remove("enumVal");
+ return this;
+ }
+
+ private Builder enumVal(String __value) {
+ return enumVal(EnumVal.Enum.valueOf(__value));
+ }
+
+ public Builder enumwithdef(Enumwithdef.Enum __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ enumwithdef = __value;
+ return this;
+ }
+
+ private Builder enumwithdef(String __value) {
+ return enumwithdef(Enumwithdef.Enum.valueOf(__value));
+ }
+
+ public Builder refVal(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ refVal = __value;
+ __uninitialized.remove("refVal");
+ return this;
+ }
+
+
+ public Builder refwithdef(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ refwithdef = __value;
+ return this;
+ }
+
+
+ public Builder fileVal(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ fileVal = __value;
+ __uninitialized.remove("fileVal");
+ return this;
+ }
+
+
+ public Builder pathVal(FileReference __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ pathVal = __value;
+ __uninitialized.remove("pathVal");
+ return this;
+ }
+
+
+ public Builder boolarr(Boolean __value) {
+ boolarr.add(__value);
+ return this;
+ }
+
+ public Builder boolarr(Collection<Boolean> __values) {
+ boolarr.addAll(__values);
+ return this;
+ }
+
+ private Builder boolarr(String __value) {
+ return boolarr(Boolean.valueOf(__value));
+ }
+
+ public Builder intarr(Integer __value) {
+ intarr.add(__value);
+ return this;
+ }
+
+ public Builder intarr(Collection<Integer> __values) {
+ intarr.addAll(__values);
+ return this;
+ }
+
+ private Builder intarr(String __value) {
+ return intarr(Integer.valueOf(__value));
+ }
+
+ public Builder longarr(Long __value) {
+ longarr.add(__value);
+ return this;
+ }
+
+ public Builder longarr(Collection<Long> __values) {
+ longarr.addAll(__values);
+ return this;
+ }
+
+ private Builder longarr(String __value) {
+ return longarr(Long.valueOf(__value));
+ }
+
+ public Builder doublearr(Double __value) {
+ doublearr.add(__value);
+ return this;
+ }
+
+ public Builder doublearr(Collection<Double> __values) {
+ doublearr.addAll(__values);
+ return this;
+ }
+
+ private Builder doublearr(String __value) {
+ return doublearr(Double.valueOf(__value));
+ }
+
+ public Builder stringarr(String __value) {
+ stringarr.add(__value);
+ return this;
+ }
+
+ public Builder stringarr(Collection<String> __values) {
+ stringarr.addAll(__values);
+ return this;
+ }
+
+ public Builder enumarr(Enumarr.Enum __value) {
+ enumarr.add(__value);
+ return this;
+ }
+
+ public Builder enumarr(Collection<Enumarr.Enum> __values) {
+ enumarr.addAll(__values);
+ return this;
+ }
+
+ private Builder enumarr(String __value) {
+ return enumarr(Enumarr.Enum.valueOf(__value));
+ }
+
+ public Builder refarr(String __value) {
+ refarr.add(__value);
+ return this;
+ }
+
+ public Builder refarr(Collection<String> __values) {
+ refarr.addAll(__values);
+ return this;
+ }
+
+ public Builder filearr(String __value) {
+ filearr.add(__value);
+ return this;
+ }
+
+ public Builder filearr(Collection<String> __values) {
+ filearr.addAll(__values);
+ return this;
+ }
+
+ public Builder pathArr(FileReference __value) {
+ pathArr.add(__value);
+ return this;
+ }
+
+ public Builder pathArr(Collection<FileReference> __values) {
+ pathArr.addAll(__values);
+ return this;
+ }
+
+ public Builder intMap(String __key, Integer __value) {
+ intMap.put(__key, __value);
+ return this;
+ }
+
+ public Builder intMap(Map<String, Integer> __values) {
+ intMap.putAll(__values);
+ return this;
+ }
+
+ private Builder intMap(String __key, String __value) {
+ return intMap(__key, Integer.valueOf(__value));
+ }
+
+ public Builder pathMap(String __key, String __value) {
+ pathMap.put(__key, __value);
+ return this;
+ }
+
+ public Builder pathMap(Map<String, String> __values) {
+ pathMap.putAll(__values);
+ return this;
+ }
+
+ public Builder basic_struct(Basic_struct.Builder __builder) {
+ basic_struct = __builder;
+ return this;
+ }
+
+ public Builder struct_of_struct(Struct_of_struct.Builder __builder) {
+ struct_of_struct = __builder;
+ return this;
+ }
+
+ /**
+ * Add the given builder to this builder's list of MyArray builders
+ * @param __builder a builder
+ * @return this builder
+ */
+ public Builder myArray(MyArray.Builder __builder) {
+ myArray.add(__builder);
+ return this;
+ }
+
+ /**
+ * Set the given list as this builder's list of MyArray builders
+ * @param __builders a list of builders
+ * @return this builder
+ */
+ public Builder myArray(List<MyArray.Builder> __builders) {
+ myArray = __builders;
+ return this;
+ }
+
+ public Builder myMap(String __key, MyMap.Builder __value) {
+ myMap.put(__key, __value);
+ return this;
+ }
+
+ public Builder myMap(Map<String, MyMap.Builder> __values) {
+ myMap.putAll(__values);
+ return this;
+ }
+
+ @java.lang.Override
+ public final boolean dispatchGetConfig(ConfigInstance.Producer producer) {
+ if (producer instanceof Producer) {
+ ((Producer)producer).getConfig(this);
+ return true;
+ }
+ return false;
+ }
+
+ @java.lang.Override
+ public final String getDefMd5() { return CONFIG_DEF_MD5; }
+ @java.lang.Override
+ public final String getDefName() { return CONFIG_DEF_NAME; }
+ @java.lang.Override
+ public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }
+ }
+
+ // Some random bool without a default value. These comments exist to check
+ // that comment parsing works.e
+ private final BooleanNode boolVal;
+ // A bool with a default value set.
+ private final BooleanNode bool_with_def;
+ private final IntegerNode intVal;
+ private final IntegerNode intWithDef;
+ private final LongNode longVal;
+ private final LongNode longWithDef;
+ private final DoubleNode doubleVal;
+ private final DoubleNode double_with_def;
+ // Another comment
+ private final StringNode stringVal;
+ private final StringNode stringwithdef;
+ private final EnumVal enumVal;
+ private final Enumwithdef enumwithdef;
+ private final ReferenceNode refVal;
+ private final ReferenceNode refwithdef;
+ private final FileNode fileVal;
+ private final PathNode pathVal;
+ private final LeafNodeVector<Boolean, BooleanNode> boolarr;
+ private final LeafNodeVector<Integer, IntegerNode> intarr;
+ private final LeafNodeVector<Long, LongNode> longarr;
+ private final LeafNodeVector<Double, DoubleNode> doublearr;
+ private final LeafNodeVector<String, StringNode> stringarr;
+ private final LeafNodeVector<Enumarr.Enum, Enumarr> enumarr;
+ private final LeafNodeVector<String, ReferenceNode> refarr;
+ private final LeafNodeVector<FileReference, FileNode> filearr;
+ private final LeafNodeVector<Path, PathNode> pathArr;
+ private final Map<String, IntegerNode> intMap;
+ private final Map<String, FileNode> pathMap;
+ private final Basic_struct basic_struct;
+ private final Struct_of_struct struct_of_struct;
+ private final InnerNodeVector<MyArray> myArray;
+ private final Map<String, MyMap> myMap;
+
+ public AllfeaturesConfig(Builder builder) {
+ this(builder, true);
+ }
+
+ private AllfeaturesConfig(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures must be initialized: " + builder.__uninitialized);
+
+ boolVal = (builder.boolVal == null) ?
+ new BooleanNode() : new BooleanNode(builder.boolVal);
+ bool_with_def = (builder.bool_with_def == null) ?
+ new BooleanNode(false) : new BooleanNode(builder.bool_with_def);
+ intVal = (builder.intVal == null) ?
+ new IntegerNode() : new IntegerNode(builder.intVal);
+ intWithDef = (builder.intWithDef == null) ?
+ new IntegerNode(-545) : new IntegerNode(builder.intWithDef);
+ longVal = (builder.longVal == null) ?
+ new LongNode() : new LongNode(builder.longVal);
+ longWithDef = (builder.longWithDef == null) ?
+ new LongNode(1234567890123L) : new LongNode(builder.longWithDef);
+ doubleVal = (builder.doubleVal == null) ?
+ new DoubleNode() : new DoubleNode(builder.doubleVal);
+ double_with_def = (builder.double_with_def == null) ?
+ new DoubleNode(-6.43D) : new DoubleNode(builder.double_with_def);
+ stringVal = (builder.stringVal == null) ?
+ new StringNode() : new StringNode(builder.stringVal);
+ stringwithdef = (builder.stringwithdef == null) ?
+ new StringNode("foobar#notacomment") : new StringNode(builder.stringwithdef);
+ enumVal = (builder.enumVal == null) ?
+ new EnumVal() : new EnumVal(builder.enumVal);
+ enumwithdef = (builder.enumwithdef == null) ?
+ new Enumwithdef(Enumwithdef.BAR2) : new Enumwithdef(builder.enumwithdef);
+ refVal = (builder.refVal == null) ?
+ new ReferenceNode() : new ReferenceNode(builder.refVal);
+ refwithdef = (builder.refwithdef == null) ?
+ new ReferenceNode(":parent:") : new ReferenceNode(builder.refwithdef);
+ fileVal = (builder.fileVal == null) ?
+ new FileNode() : new FileNode(builder.fileVal);
+ pathVal = (builder.pathVal == null) ?
+ new PathNode() : new PathNode(builder.pathVal);
+ boolarr = new LeafNodeVector<>(builder.boolarr, new BooleanNode());
+ intarr = new LeafNodeVector<>(builder.intarr, new IntegerNode());
+ longarr = new LeafNodeVector<>(builder.longarr, new LongNode());
+ doublearr = new LeafNodeVector<>(builder.doublearr, new DoubleNode());
+ stringarr = new LeafNodeVector<>(builder.stringarr, new StringNode());
+ enumarr = new LeafNodeVector<>(builder.enumarr, new Enumarr());
+ refarr = new LeafNodeVector<>(builder.refarr, new ReferenceNode());
+ filearr = LeafNodeVector.createFileNodeVector(builder.filearr);
+ pathArr = LeafNodeVector.createPathNodeVector(builder.pathArr);
+ intMap = LeafNodeMaps.asNodeMap(builder.intMap, new IntegerNode());
+ pathMap = LeafNodeMaps.asFileNodeMap(builder.pathMap);
+ basic_struct = new Basic_struct(builder.basic_struct, throwIfUninitialized);
+ struct_of_struct = new Struct_of_struct(builder.struct_of_struct, throwIfUninitialized);
+ myArray = MyArray.createVector(builder.myArray);
+ myMap = MyMap.createMap(builder.myMap);
+ }
+
+ /**
+ * @return allfeatures.boolVal
+ */
+ public boolean boolVal() {
+ return boolVal.value();
+ }
+
+ /**
+ * @return allfeatures.bool_with_def
+ */
+ public boolean bool_with_def() {
+ return bool_with_def.value();
+ }
+
+ /**
+ * @return allfeatures.intVal
+ */
+ public int intVal() {
+ return intVal.value();
+ }
+
+ /**
+ * @return allfeatures.intWithDef
+ */
+ public int intWithDef() {
+ return intWithDef.value();
+ }
+
+ /**
+ * @return allfeatures.longVal
+ */
+ public long longVal() {
+ return longVal.value();
+ }
+
+ /**
+ * @return allfeatures.longWithDef
+ */
+ public long longWithDef() {
+ return longWithDef.value();
+ }
+
+ /**
+ * @return allfeatures.doubleVal
+ */
+ public double doubleVal() {
+ return doubleVal.value();
+ }
+
+ /**
+ * @return allfeatures.double_with_def
+ */
+ public double double_with_def() {
+ return double_with_def.value();
+ }
+
+ /**
+ * @return allfeatures.stringVal
+ */
+ public String stringVal() {
+ return stringVal.value();
+ }
+
+ /**
+ * @return allfeatures.stringwithdef
+ */
+ public String stringwithdef() {
+ return stringwithdef.value();
+ }
+
+ /**
+ * @return allfeatures.enumVal
+ */
+ public EnumVal.Enum enumVal() {
+ return enumVal.value();
+ }
+
+ /**
+ * @return allfeatures.enumwithdef
+ */
+ public Enumwithdef.Enum enumwithdef() {
+ return enumwithdef.value();
+ }
+
+ /**
+ * @return allfeatures.refVal
+ */
+ public String refVal() {
+ return refVal.value();
+ }
+
+ /**
+ * @return allfeatures.refwithdef
+ */
+ public String refwithdef() {
+ return refwithdef.value();
+ }
+
+ /**
+ * @return allfeatures.fileVal
+ */
+ public FileReference fileVal() {
+ return fileVal.value();
+ }
+
+ /**
+ * @return allfeatures.pathVal
+ */
+ public Path pathVal() {
+ return pathVal.value();
+ }
+
+ /**
+ * @return allfeatures.boolarr[]
+ */
+ public List<Boolean> boolarr() {
+ return boolarr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.boolarr[]
+ */
+ public boolean boolarr(int i) {
+ return boolarr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.intarr[]
+ */
+ public List<Integer> intarr() {
+ return intarr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.intarr[]
+ */
+ public int intarr(int i) {
+ return intarr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.longarr[]
+ */
+ public List<Long> longarr() {
+ return longarr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.longarr[]
+ */
+ public long longarr(int i) {
+ return longarr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.doublearr[]
+ */
+ public List<Double> doublearr() {
+ return doublearr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.doublearr[]
+ */
+ public double doublearr(int i) {
+ return doublearr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.stringarr[]
+ */
+ public List<String> stringarr() {
+ return stringarr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.stringarr[]
+ */
+ public String stringarr(int i) {
+ return stringarr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.enumarr[]
+ */
+ public List<Enumarr.Enum> enumarr() {
+ return enumarr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.enumarr[]
+ */
+ public Enumarr.Enum enumarr(int i) {
+ return enumarr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.refarr[]
+ */
+ public List<String> refarr() {
+ return refarr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.refarr[]
+ */
+ public String refarr(int i) {
+ return refarr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.filearr[]
+ */
+ public List<FileReference> filearr() {
+ return filearr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.filearr[]
+ */
+ public FileReference filearr(int i) {
+ return filearr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.pathArr[]
+ */
+ public List<Path> pathArr() {
+ return pathArr.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.pathArr[]
+ */
+ public Path pathArr(int i) {
+ return pathArr.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.intMap{}
+ */
+ public Map<String, Integer> intMap() {
+ return LeafNodeMaps.asValueMap(intMap);
+ }
+
+ /**
+ * @param key the key of the value to return
+ * @return allfeatures.intMap{}
+ */
+ public int intMap(String key) {
+ return intMap.get(key).value();
+ }
+
+ /**
+ * @return allfeatures.pathMap{}
+ */
+ public Map<String, FileReference> pathMap() {
+ return LeafNodeMaps.asValueMap(pathMap);
+ }
+
+ /**
+ * @param key the key of the value to return
+ * @return allfeatures.pathMap{}
+ */
+ public FileReference pathMap(String key) {
+ return pathMap.get(key).value();
+ }
+
+ /**
+ * @return allfeatures.basic_struct
+ */
+ public Basic_struct basic_struct() {
+ return basic_struct;
+ }
+
+ /**
+ * @return allfeatures.struct_of_struct
+ */
+ public Struct_of_struct struct_of_struct() {
+ return struct_of_struct;
+ }
+
+ /**
+ * @return allfeatures.myArray[]
+ */
+ public List<MyArray> myArray() {
+ return myArray;
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.myArray[]
+ */
+ public MyArray myArray(int i) {
+ return myArray.get(i);
+ }
+
+ /**
+ * @return allfeatures.myMap{}
+ */
+ public Map<String, MyMap> myMap() {
+ return Collections.unmodifiableMap(myMap);
+ }
+
+ /**
+ * @param key the key of the value to return
+ * @return allfeatures.myMap{}
+ */
+ public MyMap myMap(String key) {
+ return myMap.get(key);
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(AllfeaturesConfig newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("allfeatures");
+ return changes;
+ }
+
+ private static boolean containsFieldsFlaggedWithRestart() {
+ return false;
+ }
+
+ /**
+ * This class represents allfeatures.enumVal
+ */
+ public final static class EnumVal extends EnumNode<EnumVal.Enum> {
+
+ public EnumVal(){
+ this.value = null;
+ }
+
+ public EnumVal(Enum enumValue) {
+ super(enumValue != null);
+ this.value = enumValue;
+ }
+
+ public enum Enum {FOO, BAR, FOOBAR}
+ public final static Enum FOO = Enum.FOO;
+ public final static Enum BAR = Enum.BAR;
+ public final static Enum FOOBAR = Enum.FOOBAR;
+
+ @Override
+ protected boolean doSetValue(@NonNull String name) {
+ try {
+ value = Enum.valueOf(name);
+ return true;
+ } catch (IllegalArgumentException e) {
+ }
+ return false;
+ }
+ }
+
+ /**
+ * This class represents allfeatures.enumwithdef
+ */
+ public final static class Enumwithdef extends EnumNode<Enumwithdef.Enum> {
+
+ public Enumwithdef(){
+ this.value = null;
+ }
+
+ public Enumwithdef(Enum enumValue) {
+ super(enumValue != null);
+ this.value = enumValue;
+ }
+
+ public enum Enum {FOO2, BAR2, FOOBAR2}
+ public final static Enum FOO2 = Enum.FOO2;
+ public final static Enum BAR2 = Enum.BAR2;
+ public final static Enum FOOBAR2 = Enum.FOOBAR2;
+
+ @Override
+ protected boolean doSetValue(@NonNull String name) {
+ try {
+ value = Enum.valueOf(name);
+ return true;
+ } catch (IllegalArgumentException e) {
+ }
+ return false;
+ }
+ }
+
+ /**
+ * This class represents allfeatures.enumarr[]
+ */
+ public final static class Enumarr extends EnumNode<Enumarr.Enum> {
+
+ public Enumarr(){
+ this.value = null;
+ }
+
+ public Enumarr(Enum enumValue) {
+ super(enumValue != null);
+ this.value = enumValue;
+ }
+
+ public enum Enum {ARRAY, VALUES}
+ public final static Enum ARRAY = Enum.ARRAY;
+ public final static Enum VALUES = Enum.VALUES;
+
+ @Override
+ protected boolean doSetValue(@NonNull String name) {
+ try {
+ value = Enum.valueOf(name);
+ return true;
+ } catch (IllegalArgumentException e) {
+ }
+ return false;
+ }
+ }
+
+ /**
+ * This class represents allfeatures.basic_struct
+ */
+ public final static class Basic_struct extends InnerNode {
+
+ public static class Builder implements ConfigBuilder {
+ private Set<String> __uninitialized = new HashSet<String>();
+
+ private String foo = null;
+ private Integer bar = null;
+
+ public Builder() { }
+
+ public Builder(Basic_struct config) {
+ foo(config.foo());
+ bar(config.bar());
+ }
+
+ private Builder override(Builder __superior) {
+ if (__superior.foo != null)
+ foo(__superior.foo);
+ if (__superior.bar != null)
+ bar(__superior.bar);
+ return this;
+ }
+
+ public Builder foo(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ foo = __value;
+ return this;
+ }
+
+
+ public Builder bar(int __value) {
+ bar = __value;
+ return this;
+ }
+
+ private Builder bar(String __value) {
+ return bar(Integer.valueOf(__value));
+ }
+ }
+
+ // A basic struct
+ private final StringNode foo;
+ private final IntegerNode bar;
+
+ public Basic_struct(Builder builder) {
+ this(builder, true);
+ }
+
+ private Basic_struct(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures.basic_struct must be initialized: " + builder.__uninitialized);
+
+ foo = (builder.foo == null) ?
+ new StringNode("foo") : new StringNode(builder.foo);
+ bar = (builder.bar == null) ?
+ new IntegerNode(0) : new IntegerNode(builder.bar);
+ }
+
+ /**
+ * @return allfeatures.basic_struct.foo
+ */
+ public String foo() {
+ return foo.value();
+ }
+
+ /**
+ * @return allfeatures.basic_struct.bar
+ */
+ public int bar() {
+ return bar.value();
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(Basic_struct newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("basic_struct");
+ return changes;
+ }
+ }
+
+ /**
+ * This class represents allfeatures.struct_of_struct
+ */
+ public final static class Struct_of_struct extends InnerNode {
+
+ public static class Builder implements ConfigBuilder {
+ private Set<String> __uninitialized = new HashSet<String>();
+
+ public Inner0.Builder inner0 = new Inner0.Builder();
+ public Inner1.Builder inner1 = new Inner1.Builder();
+
+ public Builder() { }
+
+ public Builder(Struct_of_struct config) {
+ inner0(new Inner0.Builder(config.inner0()));
+ inner1(new Inner1.Builder(config.inner1()));
+ }
+
+ private Builder override(Builder __superior) {
+ inner0(inner0.override(__superior.inner0));
+ inner1(inner1.override(__superior.inner1));
+ return this;
+ }
+
+ public Builder inner0(Inner0.Builder __builder) {
+ inner0 = __builder;
+ return this;
+ }
+
+ public Builder inner1(Inner1.Builder __builder) {
+ inner1 = __builder;
+ return this;
+ }
+ }
+
+ private final Inner0 inner0;
+ private final Inner1 inner1;
+
+ public Struct_of_struct(Builder builder) {
+ this(builder, true);
+ }
+
+ private Struct_of_struct(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures.struct_of_struct must be initialized: " + builder.__uninitialized);
+
+ inner0 = new Inner0(builder.inner0, throwIfUninitialized);
+ inner1 = new Inner1(builder.inner1, throwIfUninitialized);
+ }
+
+ /**
+ * @return allfeatures.struct_of_struct.inner0
+ */
+ public Inner0 inner0() {
+ return inner0;
+ }
+
+ /**
+ * @return allfeatures.struct_of_struct.inner1
+ */
+ public Inner1 inner1() {
+ return inner1;
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(Struct_of_struct newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("struct_of_struct");
+ return changes;
+ }
+
+ /**
+ * This class represents allfeatures.struct_of_struct.inner0
+ */
+ public final static class Inner0 extends InnerNode {
+
+ public static class Builder implements ConfigBuilder {
+ private Set<String> __uninitialized = new HashSet<String>();
+
+ private String name = null;
+ private Integer index = null;
+
+ public Builder() { }
+
+ public Builder(Inner0 config) {
+ name(config.name());
+ index(config.index());
+ }
+
+ private Builder override(Builder __superior) {
+ if (__superior.name != null)
+ name(__superior.name);
+ if (__superior.index != null)
+ index(__superior.index);
+ return this;
+ }
+
+ public Builder name(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ name = __value;
+ return this;
+ }
+
+
+ public Builder index(int __value) {
+ index = __value;
+ return this;
+ }
+
+ private Builder index(String __value) {
+ return index(Integer.valueOf(__value));
+ }
+ }
+
+ // A struct of struct
+ private final StringNode name;
+ private final IntegerNode index;
+
+ public Inner0(Builder builder) {
+ this(builder, true);
+ }
+
+ private Inner0(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures.struct_of_struct.inner0 must be initialized: " + builder.__uninitialized);
+
+ name = (builder.name == null) ?
+ new StringNode("inner0") : new StringNode(builder.name);
+ index = (builder.index == null) ?
+ new IntegerNode(0) : new IntegerNode(builder.index);
+ }
+
+ /**
+ * @return allfeatures.struct_of_struct.inner0.name
+ */
+ public String name() {
+ return name.value();
+ }
+
+ /**
+ * @return allfeatures.struct_of_struct.inner0.index
+ */
+ public int index() {
+ return index.value();
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(Inner0 newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("inner0");
+ return changes;
+ }
+ }
+
+ /**
+ * This class represents allfeatures.struct_of_struct.inner1
+ */
+ public final static class Inner1 extends InnerNode {
+
+ public static class Builder implements ConfigBuilder {
+ private Set<String> __uninitialized = new HashSet<String>();
+
+ private String name = null;
+ private Integer index = null;
+
+ public Builder() { }
+
+ public Builder(Inner1 config) {
+ name(config.name());
+ index(config.index());
+ }
+
+ private Builder override(Builder __superior) {
+ if (__superior.name != null)
+ name(__superior.name);
+ if (__superior.index != null)
+ index(__superior.index);
+ return this;
+ }
+
+ public Builder name(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ name = __value;
+ return this;
+ }
+
+
+ public Builder index(int __value) {
+ index = __value;
+ return this;
+ }
+
+ private Builder index(String __value) {
+ return index(Integer.valueOf(__value));
+ }
+ }
+
+ private final StringNode name;
+ private final IntegerNode index;
+
+ public Inner1(Builder builder) {
+ this(builder, true);
+ }
+
+ private Inner1(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures.struct_of_struct.inner1 must be initialized: " + builder.__uninitialized);
+
+ name = (builder.name == null) ?
+ new StringNode("inner1") : new StringNode(builder.name);
+ index = (builder.index == null) ?
+ new IntegerNode(1) : new IntegerNode(builder.index);
+ }
+
+ /**
+ * @return allfeatures.struct_of_struct.inner1.name
+ */
+ public String name() {
+ return name.value();
+ }
+
+ /**
+ * @return allfeatures.struct_of_struct.inner1.index
+ */
+ public int index() {
+ return index.value();
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(Inner1 newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("inner1");
+ return changes;
+ }
+ }
+ }
+
+ /**
+ * This class represents allfeatures.myArray[]
+ */
+ public final static class MyArray extends InnerNode {
+
+ public static class Builder implements ConfigBuilder {
+ private Set<String> __uninitialized = new HashSet<String>(Arrays.asList(
+ "refVal"
+ ));
+
+ private Integer intVal = null;
+ public List<String> stringVal = new ArrayList<>();
+ private EnumVal.Enum enumVal = null;
+ private String refVal = null;
+ public List<AnotherArray.Builder> anotherArray = new ArrayList<>();
+
+ public Builder() { }
+
+ public Builder(MyArray config) {
+ intVal(config.intVal());
+ stringVal(config.stringVal());
+ enumVal(config.enumVal());
+ refVal(config.refVal());
+ for (AnotherArray a : config.anotherArray()) {
+ anotherArray(new AnotherArray.Builder(a));
+ }
+ }
+
+ private Builder override(Builder __superior) {
+ if (__superior.intVal != null)
+ intVal(__superior.intVal);
+ if (!__superior.stringVal.isEmpty())
+ stringVal.addAll(__superior.stringVal);
+ if (__superior.enumVal != null)
+ enumVal(__superior.enumVal);
+ if (__superior.refVal != null)
+ refVal(__superior.refVal);
+ if (!__superior.anotherArray.isEmpty())
+ anotherArray.addAll(__superior.anotherArray);
+ return this;
+ }
+
+ public Builder intVal(int __value) {
+ intVal = __value;
+ return this;
+ }
+
+ private Builder intVal(String __value) {
+ return intVal(Integer.valueOf(__value));
+ }
+
+ public Builder stringVal(String __value) {
+ stringVal.add(__value);
+ return this;
+ }
+
+ public Builder stringVal(Collection<String> __values) {
+ stringVal.addAll(__values);
+ return this;
+ }
+
+ public Builder enumVal(EnumVal.Enum __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ enumVal = __value;
+ return this;
+ }
+
+ private Builder enumVal(String __value) {
+ return enumVal(EnumVal.Enum.valueOf(__value));
+ }
+
+ public Builder refVal(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ refVal = __value;
+ __uninitialized.remove("refVal");
+ return this;
+ }
+
+
+ /**
+ * Add the given builder to this builder's list of AnotherArray builders
+ * @param __builder a builder
+ * @return this builder
+ */
+ public Builder anotherArray(AnotherArray.Builder __builder) {
+ anotherArray.add(__builder);
+ return this;
+ }
+
+ /**
+ * Set the given list as this builder's list of AnotherArray builders
+ * @param __builders a list of builders
+ * @return this builder
+ */
+ public Builder anotherArray(List<AnotherArray.Builder> __builders) {
+ anotherArray = __builders;
+ return this;
+ }
+ }
+
+ private final IntegerNode intVal;
+ private final LeafNodeVector<String, StringNode> stringVal;
+ private final EnumVal enumVal;
+ private final ReferenceNode refVal;
+ private final InnerNodeVector<AnotherArray> anotherArray;
+
+ public MyArray(Builder builder) {
+ this(builder, true);
+ }
+
+ private MyArray(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures.myArray[] must be initialized: " + builder.__uninitialized);
+
+ intVal = (builder.intVal == null) ?
+ new IntegerNode(14) : new IntegerNode(builder.intVal);
+ stringVal = new LeafNodeVector<>(builder.stringVal, new StringNode());
+ enumVal = (builder.enumVal == null) ?
+ new EnumVal(EnumVal.TYPE) : new EnumVal(builder.enumVal);
+ refVal = (builder.refVal == null) ?
+ new ReferenceNode() : new ReferenceNode(builder.refVal);
+ anotherArray = AnotherArray.createVector(builder.anotherArray);
+ }
+
+ /**
+ * @return allfeatures.myArray[].intVal
+ */
+ public int intVal() {
+ return intVal.value();
+ }
+
+ /**
+ * @return allfeatures.myArray[].stringVal[]
+ */
+ public List<String> stringVal() {
+ return stringVal.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.myArray[].stringVal[]
+ */
+ public String stringVal(int i) {
+ return stringVal.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.myArray[].enumVal
+ */
+ public EnumVal.Enum enumVal() {
+ return enumVal.value();
+ }
+
+ /**
+ * @return allfeatures.myArray[].refVal
+ */
+ public String refVal() {
+ return refVal.value();
+ }
+
+ /**
+ * @return allfeatures.myArray[].anotherArray[]
+ */
+ public List<AnotherArray> anotherArray() {
+ return anotherArray;
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.myArray[].anotherArray[]
+ */
+ public AnotherArray anotherArray(int i) {
+ return anotherArray.get(i);
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(MyArray newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("myArray");
+ return changes;
+ }
+
+ private static InnerNodeVector<MyArray> createVector(List<Builder> builders) {
+ List<MyArray> elems = new ArrayList<>();
+ for (Builder b : builders) {
+ elems.add(new MyArray(b));
+ }
+ return new InnerNodeVector<MyArray>(elems);
+ }
+
+ /**
+ * This class represents allfeatures.myArray[].enumVal
+ */
+ public final static class EnumVal extends EnumNode<EnumVal.Enum> {
+
+ public EnumVal(){
+ this.value = null;
+ }
+
+ public EnumVal(Enum enumValue) {
+ super(enumValue != null);
+ this.value = enumValue;
+ }
+
+ public enum Enum {INNER, ENUM, TYPE}
+ public final static Enum INNER = Enum.INNER;
+ public final static Enum ENUM = Enum.ENUM;
+ public final static Enum TYPE = Enum.TYPE;
+
+ @Override
+ protected boolean doSetValue(@NonNull String name) {
+ try {
+ value = Enum.valueOf(name);
+ return true;
+ } catch (IllegalArgumentException e) {
+ }
+ return false;
+ }
+ }
+
+ /**
+ * This class represents allfeatures.myArray[].anotherArray[]
+ */
+ public final static class AnotherArray extends InnerNode {
+
+ public static class Builder implements ConfigBuilder {
+ private Set<String> __uninitialized = new HashSet<String>();
+
+ private Integer foo = null;
+
+ public Builder() { }
+
+ public Builder(AnotherArray config) {
+ foo(config.foo());
+ }
+
+ private Builder override(Builder __superior) {
+ if (__superior.foo != null)
+ foo(__superior.foo);
+ return this;
+ }
+
+ public Builder foo(int __value) {
+ foo = __value;
+ return this;
+ }
+
+ private Builder foo(String __value) {
+ return foo(Integer.valueOf(__value));
+ }
+ }
+
+ private final IntegerNode foo;
+
+ public AnotherArray(Builder builder) {
+ this(builder, true);
+ }
+
+ private AnotherArray(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures.myArray[].anotherArray[] must be initialized: " + builder.__uninitialized);
+
+ foo = (builder.foo == null) ?
+ new IntegerNode(-4) : new IntegerNode(builder.foo);
+ }
+
+ /**
+ * @return allfeatures.myArray[].anotherArray[].foo
+ */
+ public int foo() {
+ return foo.value();
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(AnotherArray newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("anotherArray");
+ return changes;
+ }
+
+ private static InnerNodeVector<AnotherArray> createVector(List<Builder> builders) {
+ List<AnotherArray> elems = new ArrayList<>();
+ for (Builder b : builders) {
+ elems.add(new AnotherArray(b));
+ }
+ return new InnerNodeVector<AnotherArray>(elems);
+ }
+ }
+ }
+
+ /**
+ * This class represents allfeatures.myMap{}
+ */
+ public final static class MyMap extends InnerNode {
+
+ public static class Builder implements ConfigBuilder {
+ private Set<String> __uninitialized = new HashSet<String>(Arrays.asList(
+ "refVal"
+ ));
+
+ private Integer intVal = null;
+ public List<String> stringVal = new ArrayList<>();
+ private EnumVal.Enum enumVal = null;
+ private String refVal = null;
+ public List<AnotherArray.Builder> anotherArray = new ArrayList<>();
+
+ public Builder() { }
+
+ public Builder(MyMap config) {
+ intVal(config.intVal());
+ stringVal(config.stringVal());
+ enumVal(config.enumVal());
+ refVal(config.refVal());
+ for (AnotherArray a : config.anotherArray()) {
+ anotherArray(new AnotherArray.Builder(a));
+ }
+ }
+
+ private Builder override(Builder __superior) {
+ if (__superior.intVal != null)
+ intVal(__superior.intVal);
+ if (!__superior.stringVal.isEmpty())
+ stringVal.addAll(__superior.stringVal);
+ if (__superior.enumVal != null)
+ enumVal(__superior.enumVal);
+ if (__superior.refVal != null)
+ refVal(__superior.refVal);
+ if (!__superior.anotherArray.isEmpty())
+ anotherArray.addAll(__superior.anotherArray);
+ return this;
+ }
+
+ public Builder intVal(int __value) {
+ intVal = __value;
+ return this;
+ }
+
+ private Builder intVal(String __value) {
+ return intVal(Integer.valueOf(__value));
+ }
+
+ public Builder stringVal(String __value) {
+ stringVal.add(__value);
+ return this;
+ }
+
+ public Builder stringVal(Collection<String> __values) {
+ stringVal.addAll(__values);
+ return this;
+ }
+
+ public Builder enumVal(EnumVal.Enum __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ enumVal = __value;
+ return this;
+ }
+
+ private Builder enumVal(String __value) {
+ return enumVal(EnumVal.Enum.valueOf(__value));
+ }
+
+ public Builder refVal(String __value) {
+ if (__value == null) throw new IllegalArgumentException("Null value is not allowed.");
+ refVal = __value;
+ __uninitialized.remove("refVal");
+ return this;
+ }
+
+
+ /**
+ * Add the given builder to this builder's list of AnotherArray builders
+ * @param __builder a builder
+ * @return this builder
+ */
+ public Builder anotherArray(AnotherArray.Builder __builder) {
+ anotherArray.add(__builder);
+ return this;
+ }
+
+ /**
+ * Set the given list as this builder's list of AnotherArray builders
+ * @param __builders a list of builders
+ * @return this builder
+ */
+ public Builder anotherArray(List<AnotherArray.Builder> __builders) {
+ anotherArray = __builders;
+ return this;
+ }
+ }
+
+ private final IntegerNode intVal;
+ private final LeafNodeVector<String, StringNode> stringVal;
+ private final EnumVal enumVal;
+ private final ReferenceNode refVal;
+ private final InnerNodeVector<AnotherArray> anotherArray;
+
+ public MyMap(Builder builder) {
+ this(builder, true);
+ }
+
+ private MyMap(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures.myMap{} must be initialized: " + builder.__uninitialized);
+
+ intVal = (builder.intVal == null) ?
+ new IntegerNode(15) : new IntegerNode(builder.intVal);
+ stringVal = new LeafNodeVector<>(builder.stringVal, new StringNode());
+ enumVal = (builder.enumVal == null) ?
+ new EnumVal(EnumVal.ENUM) : new EnumVal(builder.enumVal);
+ refVal = (builder.refVal == null) ?
+ new ReferenceNode() : new ReferenceNode(builder.refVal);
+ anotherArray = AnotherArray.createVector(builder.anotherArray);
+ }
+
+ /**
+ * @return allfeatures.myMap{}.intVal
+ */
+ public int intVal() {
+ return intVal.value();
+ }
+
+ /**
+ * @return allfeatures.myMap{}.stringVal[]
+ */
+ public List<String> stringVal() {
+ return stringVal.asList();
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.myMap{}.stringVal[]
+ */
+ public String stringVal(int i) {
+ return stringVal.get(i).value();
+ }
+
+ /**
+ * @return allfeatures.myMap{}.enumVal
+ */
+ public EnumVal.Enum enumVal() {
+ return enumVal.value();
+ }
+
+ /**
+ * @return allfeatures.myMap{}.refVal
+ */
+ public String refVal() {
+ return refVal.value();
+ }
+
+ /**
+ * @return allfeatures.myMap{}.anotherArray[]
+ */
+ public List<AnotherArray> anotherArray() {
+ return anotherArray;
+ }
+
+ /**
+ * @param i the index of the value to return
+ * @return allfeatures.myMap{}.anotherArray[]
+ */
+ public AnotherArray anotherArray(int i) {
+ return anotherArray.get(i);
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(MyMap newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("myMap");
+ return changes;
+ }
+
+ private static Map<String, MyMap> createMap(Map<String, Builder> builders) {
+ Map<String, MyMap> ret = new LinkedHashMap<>();
+ for(String key : builders.keySet()) {
+ ret.put(key, new MyMap(builders.get(key)));
+ }
+ return Collections.unmodifiableMap(ret);
+ }
+
+ /**
+ * This class represents allfeatures.myMap{}.enumVal
+ */
+ public final static class EnumVal extends EnumNode<EnumVal.Enum> {
+
+ public EnumVal(){
+ this.value = null;
+ }
+
+ public EnumVal(Enum enumValue) {
+ super(enumValue != null);
+ this.value = enumValue;
+ }
+
+ public enum Enum {INNER, ENUM, TYPE}
+ public final static Enum INNER = Enum.INNER;
+ public final static Enum ENUM = Enum.ENUM;
+ public final static Enum TYPE = Enum.TYPE;
+
+ @Override
+ protected boolean doSetValue(@NonNull String name) {
+ try {
+ value = Enum.valueOf(name);
+ return true;
+ } catch (IllegalArgumentException e) {
+ }
+ return false;
+ }
+ }
+
+ /**
+ * This class represents allfeatures.myMap{}.anotherArray[]
+ */
+ public final static class AnotherArray extends InnerNode {
+
+ public static class Builder implements ConfigBuilder {
+ private Set<String> __uninitialized = new HashSet<String>();
+
+ private Integer foo = null;
+
+ public Builder() { }
+
+ public Builder(AnotherArray config) {
+ foo(config.foo());
+ }
+
+ private Builder override(Builder __superior) {
+ if (__superior.foo != null)
+ foo(__superior.foo);
+ return this;
+ }
+
+ public Builder foo(int __value) {
+ foo = __value;
+ return this;
+ }
+
+ private Builder foo(String __value) {
+ return foo(Integer.valueOf(__value));
+ }
+ }
+
+ private final IntegerNode foo;
+
+ public AnotherArray(Builder builder) {
+ this(builder, true);
+ }
+
+ private AnotherArray(Builder builder, boolean throwIfUninitialized) {
+ if (throwIfUninitialized && ! builder.__uninitialized.isEmpty())
+ throw new IllegalArgumentException("The following builder parameters for " +
+ "allfeatures.myMap{}.anotherArray[] must be initialized: " + builder.__uninitialized);
+
+ foo = (builder.foo == null) ?
+ new IntegerNode(-5) : new IntegerNode(builder.foo);
+ }
+
+ /**
+ * @return allfeatures.myMap{}.anotherArray[].foo
+ */
+ public int foo() {
+ return foo.value();
+ }
+
+ private ChangesRequiringRestart getChangesRequiringRestart(AnotherArray newConfig) {
+ ChangesRequiringRestart changes = new ChangesRequiringRestart("anotherArray");
+ return changes;
+ }
+
+ private static InnerNodeVector<AnotherArray> createVector(List<Builder> builders) {
+ List<AnotherArray> elems = new ArrayList<>();
+ for (Builder b : builders) {
+ elems.add(new AnotherArray(b));
+ }
+ return new InnerNodeVector<AnotherArray>(elems);
+ }
+ }
+ }
+
+}
diff --git a/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala b/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala
deleted file mode 100644
index c1a5eb2dd6a..00000000000
--- a/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala
+++ /dev/null
@@ -1,97 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.config.codegen
-
-import org.junit.Assert.assertThat
-import org.junit.Assert.assertTrue
-import org.hamcrest.CoreMatchers.is
-import java.io.StringReader
-import ConfiggenUtil.createClassName
-import JavaClassBuilder.createUniqueSymbol
-import org.junit.{Ignore, Test}
-
-/**
- * @author gjoranv
- */
-class JavaClassBuilderTest {
-
- @Ignore
- @Test
- def visual_inspection_of_generated_class() {
- val testDefinition =
- """version=1
- |namespace=test
- |p path
- |pathArr[] path
- |f file
- |fileArr[] file
- |i int default=0
- |# A long value
- |l long default=0
- |s string default=""
- |b bool
- |# An enum value
- |e enum {A, B, C}
- |intArr[] int
- |boolArr[] bool
- |enumArr[] enum {FOO, BAR}
- |intMap{} int
- |# A struct
- |# with multi-line
- |# comment and "quotes".
- |myStruct.i int
- |myStruct.s string
- |# An inner array
- |myArr[].i int
- |myArr[].newStruct.s string
- |myArr[].newStruct.b bool
- |myArr[].intArr[] int
- |# An inner map
- |myMap{}.i int
- |myMap{}.newStruct.s string
- |myMap{}.newStruct.b bool
- |myMap{}.intArr[] int
- |intMap{} int
- |""".stripMargin
-
- val parser = new DefParser("test", new StringReader(testDefinition))
- val root = parser.getTree
- val builder = new JavaClassBuilder(root, parser.getNormalizedDefinition, null, null)
- val configClass = builder.getConfigClass("TestConfig")
- print(configClass)
- }
-
- @Test
- def testCreateUniqueSymbol() {
- val testDefinition =
- """version=1
- |namespace=test
- |m int
- |n int
- """.stripMargin
- val root = new DefParser("test", new StringReader(testDefinition)).getTree
-
- assertThat(createUniqueSymbol(root, "foo"), is("f"))
- assertThat(createUniqueSymbol(root, "name"), is("na"))
- assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m"))
-
- // The basis string is not a legal return value, even if unique, to avoid multiple symbols
- // with the same name if the same basis string is given twice.
- assertTrue(createUniqueSymbol(root, "my").startsWith(ReservedWords.INTERNAL_PREFIX + "my"))
- }
-
- @Test
- def testCreateClassName() {
- assertThat(createClassName("simple"), is("SimpleConfig"))
- assertThat(createClassName("a"), is("AConfig"))
- assertThat(createClassName("a-b-c"), is("ABCConfig"))
- assertThat(createClassName("a-1-2b"), is("A12bConfig"))
- assertThat(createClassName("my-app"), is("MyAppConfig"))
- assertThat(createClassName("MyApp"), is("MyAppConfig"))
- }
-
- @Test(expected=classOf[CodegenRuntimeException])
- def testIllegalClassName() {
- createClassName("+illegal")
- }
-
-}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 64bc9868034..51882cd2266 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.component.Version;
import com.yahoo.component.Vtag;
import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.config.FileReference;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.config.application.api.ApplicationMetaData;
import com.yahoo.config.application.api.DeployLogger;
@@ -50,11 +51,15 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.UncheckedIOException;
import java.net.URI;
+import java.nio.file.attribute.BasicFileAttributes;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -68,6 +73,8 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
+import static java.nio.file.Files.readAttributes;
+
/**
* The API for managing applications.
*
@@ -271,29 +278,62 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
return true;
}
- public HttpResponse clusterControllerStatusPage(Tenant tenant, ApplicationId applicationId, String hostName, String pathSuffix) {
- Application application = getApplication(tenant, applicationId);
-
+ public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) {
// WARNING: pathSuffix may be given by the external user. Make sure no security issues arise...
// We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise
// change the hostname and port. Exposing other paths on the cluster controller should be fine.
// TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components.
String relativePath = "clustercontroller-status/" + pathSuffix;
- return httpProxy.get(application, hostName, "container-clustercontroller", relativePath);
+ return httpProxy.get(getApplication(applicationId), hostName, "container-clustercontroller", relativePath);
}
- public Long getApplicationGeneration(Tenant tenant, ApplicationId applicationId) {
- return getApplication(tenant, applicationId).getApplicationGeneration();
+ public Long getApplicationGeneration(ApplicationId applicationId) {
+ return getApplication(applicationId).getApplicationGeneration();
}
public void restart(ApplicationId applicationId, HostFilter hostFilter) {
hostProvisioner.ifPresent(provisioner -> provisioner.restart(applicationId, hostFilter));
}
- public HttpResponse filedistributionStatus(Tenant tenant, ApplicationId applicationId, Duration timeout) {
- Application application = getApplication(tenant, applicationId);
- return fileDistributionStatus.status(application, timeout);
+ public HttpResponse filedistributionStatus(ApplicationId applicationId, Duration timeout) {
+ return fileDistributionStatus.status(getApplication(applicationId), timeout);
+ }
+
+ public Set<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, boolean deleteFromDisk) {
+ if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory");
+
+ // Find all file references in use
+ Set<String> fileReferencesInUse = new HashSet<>();
+ Set<ApplicationId> applicationIds = listApplications();
+ applicationIds.forEach(applicationId -> fileReferencesInUse.addAll(getApplication(applicationId).getModel().fileReferences()
+ .stream()
+ .map(FileReference::value)
+ .collect(Collectors.toSet())));
+ log.log(LogLevel.INFO, "File references in use : " + fileReferencesInUse);
+
+ // Find those on disk that are not in use
+ Set<String> fileReferencesOnDisk = new HashSet<>();
+ File[] filesOnDisk = fileReferencesPath.listFiles();
+ if (filesOnDisk != null)
+ fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet()));
+ log.log(LogLevel.INFO, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk);
+
+ Instant instant = Instant.now().minus(Duration.ofDays(14));
+ Set<String> fileReferencesToDelete = fileReferencesOnDisk
+ .stream()
+ .filter(fileReference -> ! fileReferencesInUse.contains(fileReference))
+ .filter(fileReference -> isFileLastModifiedBefore(new File(fileReferencesPath, fileReference), instant))
+ .collect(Collectors.toSet());
+ if (deleteFromDisk) {
+ log.log(LogLevel.INFO, "Will delete file references not in use: " + fileReferencesToDelete);
+ fileReferencesToDelete.forEach(fileReference -> {
+ File file = new File(fileReferencesPath, fileReference);
+ if ( ! IOUtils.recursiveDeleteDir(file))
+ log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath());
+ });
+ }
+ return fileReferencesToDelete;
}
public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) {
@@ -301,22 +341,37 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode);
}
- private Application getApplication(Tenant tenant, ApplicationId applicationId) {
+ private Application getApplication(ApplicationId applicationId) {
+ Tenant tenant = tenantRepository.getTenant(applicationId.tenant());
long sessionId = getSessionIdForApplication(tenant, applicationId);
RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId, 0);
return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty(), clock.instant());
}
+ private Set<ApplicationId> listApplications() {
+ return tenantRepository.getAllTenants().stream()
+ .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream())
+ .collect(Collectors.toSet());
+ }
+
+ private boolean isFileLastModifiedBefore(File fileReference, Instant instant) {
+ BasicFileAttributes fileAttributes;
+ try {
+ fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class);
+ return fileAttributes.lastModifiedTime().toInstant().isBefore(instant);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
// ---------------- Convergence ----------------------------------------------------------------
- public HttpResponse serviceConvergenceCheck(Tenant tenant, ApplicationId applicationId, String hostname, URI uri) {
- Application application = getApplication(tenant, applicationId);
- return convergeChecker.serviceConvergenceCheck(application, hostname, uri);
+ public HttpResponse serviceConvergenceCheck(ApplicationId applicationId, String hostname, URI uri) {
+ return convergeChecker.serviceConvergenceCheck(getApplication(applicationId), hostname, uri);
}
- public HttpResponse serviceListToCheckForConfigConvergence(Tenant tenant, ApplicationId applicationId, URI uri) {
- Application application = getApplication(tenant, applicationId);
- return convergeChecker.serviceListToCheckForConfigConvergence(application, uri);
+ public HttpResponse serviceListToCheckForConfigConvergence(ApplicationId applicationId, URI uri) {
+ return convergeChecker.serviceListToCheckForConfigConvergence(getApplication(applicationId), uri);
}
// ---------------- Session operations ----------------------------------------------------------------
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java
index 33718774228..72a470cf937 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java
@@ -17,8 +17,7 @@ import java.util.List;
* Config server db is the maintainer of the serverdb directory containing def files and the file system sessions.
* See also {@link com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs} which maintains directories per tenant.
*
- * @author lulf
- * @since 5.1
+ * @author Ulf Lilleengen
*/
public class ConfigServerDB {
private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigServerDB.class.getName());
@@ -41,17 +40,13 @@ public class ConfigServerDB {
}
}
- public static ConfigServerDB createTestConfigServerDb(String dbDir, String definitionsDir) {
- return new ConfigServerDB(new ConfigserverConfig(new ConfigserverConfig.Builder()
- .configServerDBDir(dbDir)
- .configDefinitionsDir(definitionsDir)));
- }
-
// The config definitions shipped with Vespa
public File classes() { return new File(Defaults.getDefaults().underVespaHome(configserverConfig.configDefinitionsDir()));}
public File serverdefs() { return new File(serverDB, "serverdefs"); }
+ public File path() { return serverDB; }
+
public static void createDirectory(File d) {
if (d.exists()) {
if (!d.isDirectory()) {
@@ -81,7 +76,4 @@ public class ConfigServerDB {
}
}
- public ConfigserverConfig getConfigserverConfig() {
- return configserverConfig;
- }
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java
index b6e8f013fd1..6828204b17c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java
@@ -27,7 +27,6 @@ public interface GlobalComponentRegistry {
Curator getCurator();
ConfigCurator getConfigCurator();
Metrics getMetrics();
- ConfigServerDB getServerDB();
SessionPreparer getSessionPreparer();
ConfigserverConfig getConfigserverConfig();
TenantListener getTenantListener();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java
index d0b830aceaa..88f54e569df 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java
@@ -31,7 +31,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry
private final ConfigCurator configCurator;
private final Metrics metrics;
private final ModelFactoryRegistry modelFactoryRegistry;
- private final ConfigServerDB serverDB;
private final SessionPreparer sessionPreparer;
private final RpcServer rpcServer;
private final ConfigserverConfig configserverConfig;
@@ -47,7 +46,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry
ConfigCurator configCurator,
Metrics metrics,
ModelFactoryRegistry modelFactoryRegistry,
- ConfigServerDB serverDB,
SessionPreparer sessionPreparer,
RpcServer rpcServer,
ConfigserverConfig configserverConfig,
@@ -61,7 +59,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry
this.configCurator = configCurator;
this.metrics = metrics;
this.modelFactoryRegistry = modelFactoryRegistry;
- this.serverDB = serverDB;
this.sessionPreparer = sessionPreparer;
this.rpcServer = rpcServer;
this.configserverConfig = configserverConfig;
@@ -80,8 +77,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry
@Override
public Metrics getMetrics() { return metrics; }
@Override
- public ConfigServerDB getServerDB() { return serverDB; }
- @Override
public SessionPreparer getSessionPreparer() { return sessionPreparer; }
@Override
public ConfigserverConfig getConfigserverConfig() { return configserverConfig; }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java
index 93cd68f6dd6..293f35558cb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java
@@ -1,7 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.deploy;
-import com.google.common.io.Files;
+import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.config.provision.TenantName;
import com.yahoo.io.IOUtils;
import com.yahoo.path.Path;
@@ -12,23 +12,24 @@ import java.io.File;
/*
* Holds file system directories for a tenant
*
- * @author tonytv
+ * @author Tony Vaagenes
*/
public class TenantFileSystemDirs {
private final File serverDB;
private final TenantName tenant;
+ public TenantFileSystemDirs(ConfigserverConfig configserverConfig, TenantName tenant) {
+ this(new ConfigServerDB(configserverConfig).path(), tenant);
+ }
+
+ // For testing
public TenantFileSystemDirs(File dir, TenantName tenant) {
this.serverDB = dir;
this.tenant = tenant;
ConfigServerDB.createDirectory(sessionsPath());
}
- public static TenantFileSystemDirs createTestDirs(TenantName tenantName) {
- return new TenantFileSystemDirs(Files.createTempDir(), tenantName);
- }
-
public File sessionsPath() {
return new File(serverDB, Path.fromString("tenants").append(tenant.value()).append("sessions").getRelative());
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
index 1c2c24cc7bb..df2287c64cb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
@@ -26,10 +26,6 @@ public class FileDirectory {
private static final Logger log = Logger.getLogger(FileDirectory.class.getName());
private final File root;
- public FileDirectory() {
- this(FileDistribution.getDefaultFileDBPath());
- }
-
public FileDirectory(File rootDir) {
root = rootDir;
try {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java
index 544451b8e10..2db89c2e8ed 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.filedistribution;
+import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.config.FileReference;
import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.jrt.ErrorCode;
@@ -11,7 +12,9 @@ import com.yahoo.jrt.Supervisor;
import com.yahoo.jrt.Target;
import com.yahoo.jrt.Transport;
import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.defaults.Defaults;
+import java.io.File;
import java.util.Set;
import java.util.logging.Logger;
@@ -22,12 +25,22 @@ public class FileDistributionImpl implements FileDistribution {
private final static Logger log = Logger.getLogger(FileDistributionImpl.class.getName());
private final Supervisor supervisor = new Supervisor(new Transport());
+ private final File fileReferencesDir;
+
+ public FileDistributionImpl(ConfigserverConfig configserverConfig) {
+ this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));
+ }
@Override
public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {
startDownloadingFileReferences(hostName, port, fileReferences);
}
+ @Override
+ public File getFileReferencesDir() {
+ return fileReferencesDir;
+ }
+
// Notifies config proxy which file references it should start downloading. It's OK if the call does not succeed,
// as downloading will then start synchronously when a service requests a file reference instead
private void startDownloadingFileReferences(String hostName, int port, Set<FileReference> fileReferences) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
index d6751987424..b3f3214793c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
@@ -17,8 +17,8 @@ public class FileDistributionProvider {
private final FileDistribution fileDistribution;
public FileDistributionProvider(File applicationDir, FileDistribution fileDistribution) {
- this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory())), fileDistribution);
- ensureDirExists(FileDistribution.getDefaultFileDBPath());
+ this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory(fileDistribution.getFileReferencesDir()))), fileDistribution);
+ ensureDirExists(fileDistribution.getFileReferencesDir());
}
FileDistributionProvider(FileRegistry fileRegistry, FileDistribution fileDistribution) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
index 001ef751e69..42bf269e9d2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.config.server.filedistribution;
import com.google.inject.Inject;
import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.config.FileReference;
-import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.subscription.ConfigSourceSet;
import com.yahoo.jrt.Int32Value;
import com.yahoo.jrt.Request;
@@ -17,6 +16,7 @@ import com.yahoo.vespa.config.Connection;
import com.yahoo.vespa.config.ConnectionPool;
import com.yahoo.vespa.config.JRTConnectionPool;
import com.yahoo.vespa.config.server.ConfigServerSpec;
+import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.vespa.filedistribution.CompressedFileReference;
import com.yahoo.vespa.filedistribution.FileDownloader;
import com.yahoo.vespa.filedistribution.FileReferenceData;
@@ -72,7 +72,7 @@ public class FileServer {
@Inject
public FileServer(ConfigserverConfig configserverConfig) {
- this(createConnectionPool(configserverConfig), FileDistribution.getDefaultFileDBPath());
+ this(createConnectionPool(configserverConfig), new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())));
}
// For testing only
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java
index 728f327c829..40d75d9dbac 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java
@@ -4,14 +4,23 @@ package com.yahoo.vespa.config.server.filedistribution;
import com.yahoo.config.FileReference;
import com.yahoo.config.model.api.FileDistribution;
+import java.io.File;
import java.util.Set;
/**
* @author Ulf Lilleengen
*/
-public class MockFileDBHandler implements FileDistribution {
+public class MockFileDistribution implements FileDistribution {
+ private final File fileReferencesDir;
+
+ MockFileDistribution(File fileReferencesDir) {
+ this.fileReferencesDir = fileReferencesDir;
+ }
@Override
public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {}
+ @Override
+ public File getFileReferencesDir() { return fileReferencesDir; }
+
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java
index b4ed2352d00..db70a51b2b4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java
@@ -4,14 +4,16 @@ package com.yahoo.vespa.config.server.filedistribution;
import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.model.application.provider.MockFileRegistry;
+import java.io.File;
+
/**
* @author Ulf Lilleengen
*/
public class MockFileDistributionProvider extends FileDistributionProvider {
public int timesCalled = 0;
- public MockFileDistributionProvider() {
- super(new MockFileRegistry(), new MockFileDBHandler());
+ public MockFileDistributionProvider(File fileReferencesDir) {
+ super(new MockFileRegistry(), new MockFileDistribution(fileReferencesDir));
}
public FileDistribution getFileDistribution() {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 42fdb16c7ca..6bca8b1c562 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -59,13 +59,13 @@ public class ApplicationHandler extends HttpHandler {
Tenant tenant = verifyTenantAndApplication(applicationId);
if (isServiceConvergeRequest(request)) {
- return applicationRepository.serviceConvergenceCheck(tenant, applicationId, getHostNameFromRequest(request), request.getUri());
+ return applicationRepository.serviceConvergenceCheck(applicationId, getHostNameFromRequest(request), request.getUri());
}
if (isClusterControllerStatusRequest(request)) {
String hostName = getHostNameFromRequest(request);
String pathSuffix = getPathSuffix(request);
- return applicationRepository.clusterControllerStatusPage(tenant, applicationId, hostName, pathSuffix);
+ return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix);
}
if (isContentRequest(request)) {
@@ -86,15 +86,15 @@ public class ApplicationHandler extends HttpHandler {
}
if (isServiceConvergeListRequest(request)) {
- return applicationRepository.serviceListToCheckForConfigConvergence(tenant, applicationId, request.getUri());
+ return applicationRepository.serviceListToCheckForConfigConvergence(applicationId, request.getUri());
}
if (isFiledistributionStatusRequest(request)) {
Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5));
- return applicationRepository.filedistributionStatus(tenant, applicationId, timeout);
+ return applicationRepository.filedistributionStatus(applicationId, timeout);
}
- return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(tenant, applicationId));
+ return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId));
}
@Override
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
index 2c46f2968ce..a08b077699c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
@@ -3,8 +3,10 @@ package com.yahoo.vespa.config.server.maintenance;
import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.provision.SystemName;
import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.session.FileDistributionFactory;
import com.yahoo.vespa.curator.Curator;
import java.time.Duration;
@@ -13,20 +15,24 @@ public class ConfigServerMaintenance extends AbstractComponent {
private final TenantsMaintainer tenantsMaintainer;
private final ZooKeeperDataMaintainer zooKeeperDataMaintainer;
+ private final FileDistributionMaintainer fileDistributionMaintainer;
@SuppressWarnings("unused") // instantiated by Dependency Injection
public ConfigServerMaintenance(ConfigserverConfig configserverConfig,
ApplicationRepository applicationRepository,
- Curator curator) {
+ Curator curator,
+ FileDistributionFactory fileDistributionFactory) {
DefaultTimes defaults = new DefaultTimes(configserverConfig);
tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval);
- zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.zookeeperDataMaintainerInterval);
+ zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.defaultInterval);
+ fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig);
}
@Override
public void deconstruct() {
tenantsMaintainer.deconstruct();
zooKeeperDataMaintainer.deconstruct();
+ fileDistributionMaintainer.deconstruct();
}
/*
@@ -37,7 +43,6 @@ public class ConfigServerMaintenance extends AbstractComponent {
private final Duration defaultInterval;
private final Duration tenantsMaintainerInterval;
- private final Duration zookeeperDataMaintainerInterval;
DefaultTimes(ConfigserverConfig configserverConfig) {
boolean isCd = configserverConfig.system().equals(SystemName.cd.name());
@@ -45,7 +50,6 @@ public class ConfigServerMaintenance extends AbstractComponent {
this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes());
// TODO: Want job control or feature flag to control when to run this, for now use a very long interval unless in CD
this.tenantsMaintainerInterval = isCd ? defaultInterval : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes());
- this.zookeeperDataMaintainerInterval = defaultInterval;
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
new file mode 100644
index 00000000000..58141a3a045
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
@@ -0,0 +1,33 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.maintenance;
+
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.defaults.Defaults;
+
+import java.io.File;
+import java.time.Duration;
+
+public class FileDistributionMaintainer extends Maintainer {
+
+ private final ApplicationRepository applicationRepository;
+ private final File fileReferencesDir;
+
+ public FileDistributionMaintainer(ApplicationRepository applicationRepository,
+ Curator curator,
+ Duration interval,
+ ConfigserverConfig configserverConfig) {
+ super(applicationRepository, curator, interval);
+ this.applicationRepository = applicationRepository;
+ this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));;
+ }
+
+
+ @Override
+ protected void maintain() {
+ // TODO: Does not delete, for now just outputs what should be deleted
+ applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, false);
+ }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java
index 852768b6937..fd57732eb30 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java
@@ -19,5 +19,6 @@ public class ZooKeeperDataMaintainer extends Maintainer {
@Override
protected void maintain() {
curator.delete(Path.fromString("/vespa/filedistribution"));
+ curator.delete(Path.fromString("/vespa/config"));
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index 551987f4dca..9d6247c5f80 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -27,6 +27,7 @@ import com.yahoo.vespa.config.server.provision.StaticProvisioner;
import java.net.URI;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -146,7 +147,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
List<MODELRESULT> allApplicationVersions = new ArrayList<>();
allApplicationVersions.add(latestModelVersion);
- if (zone().environment() == Environment.dev)
+ if (Arrays.asList(Environment.dev, Environment.test).contains(zone().environment()))
versions = keepThoseUsedOn(allocatedHosts.get(), versions);
// TODO: We use the allocated hosts from the newest version when building older model versions.
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java
index d3a74486d12..8394494adca 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java
@@ -1,6 +1,8 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.session;
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.vespa.config.server.filedistribution.FileDistributionImpl;
import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider;
@@ -14,8 +16,15 @@ import java.io.File;
@SuppressWarnings("WeakerAccess")
public class FileDistributionFactory {
+ private final ConfigserverConfig configserverConfig;
+
+ @Inject
+ public FileDistributionFactory(ConfigserverConfig configserverConfig) {
+ this.configserverConfig = configserverConfig;
+ }
+
public FileDistributionProvider createProvider(File applicationPackage) {
- return new FileDistributionProvider(applicationPackage, new FileDistributionImpl());
+ return new FileDistributionProvider(applicationPackage, new FileDistributionImpl(configserverConfig));
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java
index 69721ed01d4..ad967f49964 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java
@@ -14,9 +14,7 @@ import com.yahoo.vespa.config.server.application.ZKTenantApplications;
import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs;
import com.yahoo.vespa.config.server.monitoring.Metrics;
import com.yahoo.vespa.config.server.session.*;
-import com.yahoo.vespa.defaults.Defaults;
-import java.io.File;
import java.time.Clock;
import java.util.Collections;
@@ -162,12 +160,7 @@ public class TenantBuilder {
private void createServerDbDirs() {
if (tenantFileSystemDirs == null) {
- tenantFileSystemDirs = new TenantFileSystemDirs(new File(
- Defaults.getDefaults().underVespaHome(componentRegistry
- .getServerDB()
- .getConfigserverConfig()
- .configServerDBDir())),
- tenant);
+ tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigserverConfig(), tenant);
}
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index 058e39eea9b..f7e76c2b3bb 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -10,10 +10,9 @@ import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.Provisioner;
import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream;
-import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest;
+import com.yahoo.io.IOUtils;
+import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.server.http.SessionHandlerTest;
-import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler;
import com.yahoo.vespa.config.server.http.v2.PrepareResult;
import com.yahoo.vespa.config.server.session.PrepareParams;
import com.yahoo.vespa.config.server.tenant.Tenant;
@@ -21,14 +20,17 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.mock.MockCurator;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.util.Collections;
+import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -48,18 +50,22 @@ public class ApplicationRepositoryTest {
private Tenant tenant;
private ApplicationRepository applicationRepository;
+ private TenantRepository tenantRepository;
private TimeoutBudget timeoutBudget;
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Before
public void setup() {
Curator curator = new MockCurator();
- TenantRepository tenants = new TenantRepository(new TestComponentRegistry.Builder()
+ tenantRepository = new TenantRepository(new TestComponentRegistry.Builder()
.curator(curator)
.build());
- tenants.addTenant(tenantName);
- tenant = tenants.getTenant(tenantName);
+ tenantRepository.addTenant(tenantName);
+ tenant = tenantRepository.getTenant(tenantName);
Provisioner provisioner = new SessionHandlerTest.MockProvisioner();
- applicationRepository = new ApplicationRepository(tenants, provisioner, clock);
+ applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock);
timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60));
}
@@ -79,15 +85,15 @@ public class ApplicationRepositoryTest {
}
@Test
- public void createAndPrepareAndActivate() throws IOException {
- PrepareResult result = deployApp();
+ public void createAndPrepareAndActivate() {
+ PrepareResult result = deployApp(testApp);
assertTrue(result.configChangeActions().getRefeedActions().isEmpty());
assertTrue(result.configChangeActions().getRestartActions().isEmpty());
}
@Test
- public void deleteUnusedTenants() throws IOException {
- deployApp();
+ public void deleteUnusedTenants() {
+ deployApp(testApp);
assertTrue(applicationRepository.removeUnusedTenants().isEmpty());
applicationRepository.remove(applicationId());
assertEquals(tenantName, applicationRepository.removeUnusedTenants().iterator().next());
@@ -110,17 +116,58 @@ public class ApplicationRepositoryTest {
assertEquals(Vtag.currentVersion, ApplicationRepository.decideVersion(regularApp, Environment.perf, targetVersion));
}
+ @Test
+ public void deleteUnusedFileReferences() throws IOException {
+ File fileReferencesDir = temporaryFolder.newFolder();
+
+ // Add file reference that is not in use and should be deleted (older than 14 days)
+ File filereferenceDir = createFilereferenceOnDisk(new File(fileReferencesDir, "foo"), Instant.now().minus(Duration.ofDays(15)));
+ // Add file reference that is not in use, but should not be deleted (not older than 14 days)
+ File filereferenceDir2 = createFilereferenceOnDisk(new File(fileReferencesDir, "baz"), Instant.now());
+
+ tenantRepository.addTenant(tenantName);
+ tenant = tenantRepository.getTenant(tenantName);
+ Provisioner provisioner = new SessionHandlerTest.MockProvisioner();
+ applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock);
+ timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60));
+
+ // TODO: Deploy an app with a bundle or file that will be a file reference, too much missing in test setup to get this working now
+ PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build();
+ deployApp(new File("src/test/apps/app"), prepareParams);
+
+ boolean deleteFiles = false;
+ Set<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles);
+ assertEquals(Collections.singleton("foo"), toBeDeleted);
+ assertTrue(filereferenceDir.exists());
+ assertTrue(filereferenceDir2.exists());
+
+ deleteFiles = true;
+ toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles);
+ assertEquals(Collections.singleton("foo"), toBeDeleted);
+ assertFalse(filereferenceDir.exists());
+ assertTrue(filereferenceDir2.exists());
+ }
+
+ private File createFilereferenceOnDisk(File filereferenceDir, Instant lastModifiedTime) {
+ assertTrue(filereferenceDir.mkdir());
+ File bar = new File(filereferenceDir, "file");
+ IOUtils.writeFile(bar, Utf8.toBytes("test"));
+ assertTrue(filereferenceDir.setLastModified(lastModifiedTime.toEpochMilli()));
+ return filereferenceDir;
+ }
+
private PrepareResult prepareAndActivateApp(File application) throws IOException {
FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application);
long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, appDir.getAppDir());
return applicationRepository.prepareAndActivate(tenant, sessionId, prepareParams(), false, false, Instant.now());
}
- private PrepareResult deployApp() throws IOException {
- File file = CompressedApplicationInputStreamTest.createTarFile();
- return applicationRepository.deploy(CompressedApplicationInputStream.createFromCompressedStream(
- new FileInputStream(file), ApplicationApiHandler.APPLICATION_X_GZIP),
- prepareParams(), false, false, Instant.now());
+ private PrepareResult deployApp(File applicationPackage) {
+ return deployApp(applicationPackage, prepareParams());
+ }
+
+ private PrepareResult deployApp(File applicationPackage, PrepareParams prepareParams) {
+ return applicationRepository.deploy(applicationPackage, prepareParams);
}
private PrepareParams prepareParams() {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
index 77550b3cd35..67cc87ae223 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server;
-import com.google.common.io.Files;
import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.container.jdisc.config.HealthMonitorConfig;
import com.yahoo.container.jdisc.state.StateMonitor;
@@ -9,12 +8,12 @@ import com.yahoo.jdisc.core.SystemTimer;
import com.yahoo.vespa.config.server.deploy.DeployTester;
import com.yahoo.vespa.config.server.rpc.RpcServer;
import com.yahoo.vespa.config.server.version.VersionState;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
+import java.io.IOException;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
@@ -35,7 +34,7 @@ public class ConfigServerBootstrapTest {
@Test
public void testBootStrap() throws Exception {
- ConfigserverConfig configserverConfig = createConfigserverConfig();
+ ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig);
tester.deployApp("myApp", "4.5.6", Instant.now());
@@ -55,7 +54,7 @@ public class ConfigServerBootstrapTest {
@Test
public void testBootStrapWhenRedeploymentFails() throws Exception {
- ConfigserverConfig configserverConfig = createConfigserverConfig();
+ ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig);
tester.deployApp("myApp", "4.5.6", Instant.now());
@@ -90,8 +89,8 @@ public class ConfigServerBootstrapTest {
throw new RuntimeException(messageIfWaitingFails);
}
- private MockRpc createRpcServer(ConfigserverConfig configserverConfig) {
- return new MockRpc(configserverConfig.rpcport());
+ private MockRpc createRpcServer(ConfigserverConfig configserverConfig) throws IOException {
+ return new MockRpc(configserverConfig.rpcport(), temporaryFolder.newFolder());
}
private StateMonitor createStateMonitor() {
@@ -99,10 +98,10 @@ public class ConfigServerBootstrapTest {
new SystemTimer());
}
- private static ConfigserverConfig createConfigserverConfig() {
+ private static ConfigserverConfig createConfigserverConfig(TemporaryFolder temporaryFolder) throws IOException {
return new ConfigserverConfig(new ConfigserverConfig.Builder()
- .configServerDBDir(Files.createTempDir().getAbsolutePath())
- .configDefinitionsDir(Files.createTempDir().getAbsolutePath())
+ .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath())
+ .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath())
.hostedVespa(true)
.multitenant(true));
}
@@ -111,8 +110,8 @@ public class ConfigServerBootstrapTest {
volatile boolean isRunning = false;
- MockRpc(int port) {
- super(port);
+ MockRpc(int port, File tempDir) {
+ super(port, tempDir);
}
@Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java
index c34f6512f29..0c8a9e5c3d8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java
@@ -1,10 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server;
-import com.google.common.io.Files;
+import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.io.IOUtils;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
@@ -13,25 +15,28 @@ import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
/**
- * @author lulf
- * @since 5.1
+ * @author Ulf Lilleengen
*/
public class ConfigServerDBTest {
private ConfigServerDB serverDB;
- private File dbDir;
- private File definitionsDir;
+ private ConfigserverConfig configserverConfig;
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Before
- public void setup() {
- dbDir = Files.createTempDir();
- definitionsDir = Files.createTempDir();
- serverDB = ConfigServerDB.createTestConfigServerDb(dbDir.getAbsolutePath(), definitionsDir.getAbsolutePath());
+ public void setup() throws IOException {
+ configserverConfig = new ConfigserverConfig(
+ new ConfigserverConfig.Builder()
+ .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath())
+ .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()));
+ serverDB = new ConfigServerDB(configserverConfig);
}
private void createInitializer() throws IOException {
File existingDef = new File(serverDB.classes(), "test.def");
IOUtils.writeFile(existingDef, "hello", false);
- ConfigServerDB.createTestConfigServerDb(dbDir.getAbsolutePath(), definitionsDir.getAbsolutePath());
+ new ConfigServerDB(configserverConfig);
}
@Test
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java
index dec9dd991de..5ca3deab1fe 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java
@@ -1,11 +1,9 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server;
-import com.google.common.io.Files;
import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.config.model.NullConfigModelRegistry;
import com.yahoo.config.model.api.ConfigDefinitionRepo;
-import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.config.server.application.PermanentApplicationPackage;
import com.yahoo.vespa.config.server.filedistribution.FileServer;
@@ -22,8 +20,11 @@ import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.model.VespaModelFactory;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import java.io.IOException;
import java.util.Collections;
import static org.hamcrest.Matchers.is;
@@ -31,14 +32,12 @@ import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
/**
- * @author lulf
- * @since 5.1
+ * @author Ulf Lilleengen
*/
public class InjectedGlobalComponentRegistryTest {
private Curator curator;
private Metrics metrics;
- private ConfigServerDB serverDB;
private SessionPreparer sessionPreparer;
private ConfigserverConfig configserverConfig;
private RpcServer rpcServer;
@@ -50,33 +49,36 @@ public class InjectedGlobalComponentRegistryTest {
private ModelFactoryRegistry modelFactoryRegistry;
private Zone zone;
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Before
- public void setupRegistry() {
+ public void setupRegistry() throws IOException {
curator = new MockCurator();
ConfigCurator configCurator = ConfigCurator.create(curator);
metrics = Metrics.createTestMetrics();
modelFactoryRegistry = new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())));
configserverConfig = new ConfigserverConfig(
new ConfigserverConfig.Builder()
- .configServerDBDir(Files.createTempDir().getAbsolutePath())
- .configDefinitionsDir(Files.createTempDir().getAbsolutePath()));
- serverDB = new ConfigServerDB(configserverConfig);
+ .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath())
+ .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()));
sessionPreparer = new SessionTest.MockSessionPreparer();
- rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(),
- new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath()));
+ rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(),
+ new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(temporaryFolder.newFolder("filereferences")));
generationCounter = new SuperModelGenerationCounter(curator);
defRepo = new StaticConfigDefinitionRepo();
permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig);
hostRegistries = new HostRegistries();
HostProvisionerProvider hostProvisionerProvider = HostProvisionerProvider.withProvisioner(new SessionHandlerTest.MockProvisioner());
zone = Zone.defaultZone();
- globalComponentRegistry = new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, serverDB, sessionPreparer, rpcServer, configserverConfig, generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone);
+ globalComponentRegistry =
+ new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, sessionPreparer, rpcServer, configserverConfig,
+ generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone);
}
@Test
public void testThatAllComponentsAreSetup() {
assertThat(globalComponentRegistry.getModelFactoryRegistry(), is(modelFactoryRegistry));
- assertThat(globalComponentRegistry.getServerDB(), is(serverDB));
assertThat(globalComponentRegistry.getSessionPreparer(), is(sessionPreparer));
assertThat(globalComponentRegistry.getMetrics(), is(metrics));
assertThat(globalComponentRegistry.getCurator(), is(curator));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java
index d026989a43e..e4e45d3a014 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java
@@ -23,6 +23,7 @@ import com.yahoo.vespa.curator.mock.MockCurator;
import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.model.VespaModelFactory;
+import java.io.File;
import java.time.Clock;
import java.util.Collections;
import java.util.Optional;
@@ -37,7 +38,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry {
private final Curator curator;
private final ConfigCurator configCurator;
private final Metrics metrics;
- private final ConfigServerDB serverDB;
private final SessionPreparer sessionPreparer;
private final ConfigserverConfig configserverConfig;
private final SuperModelGenerationCounter superModelGenerationCounter;
@@ -57,7 +57,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry {
PermanentApplicationPackage permanentApplicationPackage,
FileDistributionFactory fileDistributionFactory,
SuperModelGenerationCounter superModelGenerationCounter,
- ConfigServerDB configServerDB,
HostRegistries hostRegistries,
ConfigserverConfig configserverConfig,
SessionPreparer sessionPreparer,
@@ -71,7 +70,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry {
this.configCurator = configCurator;
this.metrics = metrics;
this.configserverConfig = configserverConfig;
- this.serverDB = configServerDB;
this.reloadListener = reloadListener;
this.tenantListener = tenantListener;
this.superModelGenerationCounter = superModelGenerationCounter;
@@ -155,7 +153,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry {
final PermanentApplicationPackage permApp = this.permanentApplicationPackage
.orElse(new PermanentApplicationPackage(configserverConfig));
FileDistributionFactory fileDistributionFactory = this.fileDistributionFactory
- .orElse(new MockFileDistributionFactory());
+ .orElse(new MockFileDistributionFactory(new File(configserverConfig.fileReferencesDir())));
HostProvisionerProvider hostProvisionerProvider = hostProvisioner.isPresent() ?
HostProvisionerProvider.withProvisioner(hostProvisioner.get()) :
HostProvisionerProvider.empty();
@@ -168,7 +166,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry {
permApp,
fileDistributionFactory,
new SuperModelGenerationCounter(curator),
- new ConfigServerDB(configserverConfig),
hostRegistries, configserverConfig, sessionPreparer,
hostProvisioner, defRepo, reloadListener,
tenantListener, zone, clock);
@@ -182,8 +179,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry {
@Override
public Metrics getMetrics() { return metrics; }
@Override
- public ConfigServerDB getServerDB() { return serverDB; }
- @Override
public SessionPreparer getSessionPreparer() { return sessionPreparer; }
@Override
public ConfigserverConfig getConfigserverConfig() { return configserverConfig; }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
index 157c36d7aef..9ba8adf0aa4 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
@@ -6,7 +6,9 @@ import com.yahoo.io.IOUtils;
import com.yahoo.net.HostName;
import com.yahoo.vespa.filedistribution.FileReferenceData;
import org.junit.After;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
@@ -35,6 +37,9 @@ public class FileServerTest {
created.add(dir);
}
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Test
public void requireThatExistingFileCanBeFound() throws IOException {
createCleanDir("123");
@@ -71,10 +76,12 @@ public class FileServerTest {
}
@Test
- public void requireThatDifferentNumberOfConfigServersWork() {
+ public void requireThatDifferentNumberOfConfigServersWork() throws IOException {
// Empty connection pool in tests etc.
- ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder();
- FileServer fileServer = new FileServer(new ConfigserverConfig(builder));
+ ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder()
+ .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath())
+ .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath());
+ FileServer fileServer = createFileServer(builder);
assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize());
// Empty connection pool when only one server, no use in downloading from yourself
@@ -84,7 +91,7 @@ public class FileServerTest {
serverBuilder.port(123456);
servers.add(serverBuilder);
builder.zookeeperserver(servers);
- fileServer = new FileServer(new ConfigserverConfig(builder));
+ fileServer = createFileServer(builder);
assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize());
// connection pool of size 1 when 2 servers
@@ -93,10 +100,16 @@ public class FileServerTest {
serverBuilder2.port(123456);
servers.add(serverBuilder2);
builder.zookeeperserver(servers);
- fileServer = new FileServer(new ConfigserverConfig(builder));
+ fileServer = createFileServer(builder);
assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize());
}
+ private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException {
+ File fileReferencesDir = temporaryFolder.newFolder();
+ configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath());
+ return new FileServer(new ConfigserverConfig(configBuilder));
+ }
+
private static class FileReceiver implements FileServer.Receiver {
CompletableFuture<byte []> content;
FileReceiver(CompletableFuture<byte []> content) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java
index 2a05688c435..fba4e40000d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java
@@ -50,7 +50,9 @@ import com.yahoo.vespa.model.VespaModelFactory;
import org.hamcrest.core.Is;
import org.junit.Before;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -89,6 +91,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest {
private TenantRepository tenantRepository;
private SessionActiveHandler handler;
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Before
public void setup() {
remoteSessionRepo = new RemoteSessionRepo(tenantName);
@@ -221,9 +226,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest {
return session;
}
- private void addLocalSession(long sessionId, DeployData deployData, SessionZooKeeperClient zkc) {
+ private void addLocalSession(long sessionId, DeployData deployData, SessionZooKeeperClient zkc) throws IOException {
writeApplicationId(zkc, deployData.getApplicationName());
- TenantFileSystemDirs tenantFileSystemDirs = TenantFileSystemDirs.createTestDirs(tenantName);
+ TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(temporaryFolder.newFolder(), tenantName);
ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(testApp, deployData);
localRepo.addSession(new LocalSession(tenantName, sessionId, new SessionTest.MockSessionPreparer(),
new SessionContext(app, zkc, new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)),
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java
index 1c886adde73..ceb9b7129b4 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java
@@ -20,11 +20,13 @@ public class ZooKeeperDataMaintainerTest {
curator.create(Path.fromString("/foo"));
curator.create(Path.fromString("/vespa/bar"));
curator.create(Path.fromString("/vespa/filedistribution"));
+ curator.create(Path.fromString("/vespa/config"));
new ZooKeeperDataMaintainer(tester.applicationRepository(), curator, Duration.ofDays(1)).run();
assertTrue(curator.exists(Path.fromString("/foo")));
assertTrue(curator.exists(Path.fromString("/vespa")));
assertFalse(curator.exists(Path.fromString("/vespa/filedistribution")));
+ assertFalse(curator.exists(Path.fromString("/vespa/config")));
}
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
index 0126a9e2f29..9455798c4ea 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java
@@ -12,8 +12,11 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
import com.yahoo.vespa.config.protocol.Trace;
import com.yahoo.vespa.config.server.GetConfigContext;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -28,10 +31,13 @@ import static org.junit.Assert.assertTrue;
*/
public class DelayedConfigResponseTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Test
- public void testDelayedConfigResponses() {
+ public void testDelayedConfigResponses() throws IOException {
- MockRpc rpc = new MockRpc(13337);
+ MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder());
DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
assertThat(responses.size(), is(0));
JRTServerConfigRequest req = createRequest("foo", "md5", "myid", "mymd5", 3, 1000000, "bar");
@@ -50,9 +56,9 @@ public class DelayedConfigResponseTest {
}
@Test
- public void testDelayResponseRemove() {
+ public void testDelayResponseRemove() throws IOException {
GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId());
- MockRpc rpc = new MockRpc(13337);
+ MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder());
DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
responses.delayResponse(createRequest("foolio", "md5", "myid", "mymd5", 3, 100000, "bar"), context);
assertThat(responses.size(), is(1));
@@ -61,8 +67,8 @@ public class DelayedConfigResponseTest {
}
@Test
- public void testDelayedConfigResponse() {
- MockRpc rpc = new MockRpc(13337);
+ public void testDelayedConfigResponse() throws IOException {
+ MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder());
DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false);
assertThat(responses.size(), is(0));
assertThat(responses.toString(), is("DelayedConfigResponses. Average Size=0"));
@@ -72,7 +78,7 @@ public class DelayedConfigResponseTest {
assertThat(rpc.latestRequest, is(req));
}
- public JRTServerConfigRequest createRequest(String configName, String defMd5, String configId, String md5, long generation, long timeout, String namespace) {
+ private JRTServerConfigRequest createRequest(String configName, String defMd5, String configId, String md5, long generation, long timeout, String namespace) {
Request request = JRTClientConfigRequestV3.
createWithParams(new ConfigKey<>(configName, configId, namespace, defMd5, null), DefContent.fromList(Collections.emptyList()),
"fromHost", md5, generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED,
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java
index 50c39b74fa2..1a4d04d0323 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java
@@ -16,10 +16,11 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
import com.yahoo.vespa.config.protocol.Trace;
-import com.yahoo.vespa.config.server.rpc.GetConfigProcessor;
-import com.yahoo.vespa.config.server.rpc.MockRpc;
import com.yahoo.vespa.config.server.tenant.MockTenantProvider;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
import static org.junit.Assert.assertFalse;
import java.io.IOException;
@@ -39,9 +40,12 @@ import static org.junit.Assert.assertTrue;
*/
public class GetConfigProcessorTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Test
- public void testSentinelConfig() {
- MockRpc rpc = new MockRpc(13337, false);
+ public void testSentinelConfig() throws IOException {
+ MockRpc rpc = new MockRpc(13337, false, temporaryFolder.newFolder());
rpc.response = new MockConfigResponse("foo"); // should be a sentinel config, but it does not matter for this test
// one tenant, which has host1 assigned
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java
index 5a9735f774a..b4de201bd0b 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.config.server.rpc;
import com.yahoo.cloud.config.ConfigserverConfig;
-import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.Version;
import com.yahoo.vespa.config.protocol.ConfigResponse;
@@ -14,6 +13,7 @@ import com.yahoo.vespa.config.server.host.HostRegistries;
import com.yahoo.vespa.config.server.monitoring.Metrics;
import com.yahoo.vespa.config.server.tenant.MockTenantProvider;
+import java.io.File;
import java.util.Optional;
import java.util.concurrent.CompletionService;
@@ -36,20 +36,20 @@ public class MockRpc extends RpcServer {
public volatile JRTServerConfigRequest latestRequest = null;
- public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication) {
+ public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication, File tempDir) {
super(createConfig(port), null, Metrics.createTestMetrics(),
- new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath()));
+ new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(tempDir));
if (createDefaultTenant) {
onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication));
}
}
- public MockRpc(int port, boolean createDefaultTenant) {
- this(port, createDefaultTenant, true);
+ public MockRpc(int port, boolean createDefaultTenant, File tempDir) {
+ this(port, createDefaultTenant, true, tempDir);
}
- public MockRpc(int port) {
- this(port, true);
+ public MockRpc(int port, File tempDir) {
+ this(port, true, tempDir);
}
/** Reset fields used to assert on the calls made to this */
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
index 6bb566963c0..86412f97c20 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
@@ -48,7 +48,7 @@ public class RpcServerTest extends TestWithRpc {
testEmptyConfigHostedVespa();
}
- private void testEmptyConfigHostedVespa() throws InterruptedException {
+ private void testEmptyConfigHostedVespa() throws InterruptedException, IOException {
rpcServer.onTenantDelete(TenantName.defaultName());
rpcServer.onTenantsLoaded();
JRTClientConfigRequest clientReq = createSimpleRequest();
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java
index e022b622fb0..845e7c0f914 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.config.server.rpc;
import com.yahoo.cloud.config.ConfigserverConfig;
-import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.provision.HostLivenessTracker;
import com.yahoo.config.provision.TenantName;
import com.yahoo.jrt.Request;
@@ -20,7 +19,10 @@ import com.yahoo.vespa.config.server.monitoring.Metrics;
import com.yahoo.vespa.config.server.tenant.MockTenantProvider;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
@@ -36,8 +38,7 @@ import static org.junit.Assert.assertTrue;
/**
* Test running rpc server.
*
- * @author lulf
- * @since 5.17
+ * @author Ulf Lilleengen
*/
// TODO: Make this a Tester instead of a superclass
public class TestWithRpc {
@@ -56,8 +57,11 @@ public class TestWithRpc {
private List<Integer> allocatedPorts;
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Before
- public void setupRpc() throws InterruptedException {
+ public void setupRpc() throws InterruptedException, IOException {
allocatedPorts = new ArrayList<>();
port = allocatePort();
spec = createSpec(port);
@@ -80,7 +84,7 @@ public class TestWithRpc {
return port;
}
- protected void createAndStartRpcServer(boolean hostedVespa) {
+ protected void createAndStartRpcServer(boolean hostedVespa) throws IOException {
ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder());
rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder()
.rpcport(port)
@@ -94,7 +98,7 @@ public class TestWithRpc {
emptyNodeFlavors(),
generationCounter)),
Metrics.createTestMetrics(), new HostRegistries(),
- hostLivenessTracker, new FileServer(FileDistribution.getDefaultFileDBPath()));
+ hostLivenessTracker, new FileServer(temporaryFolder.newFolder()));
rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider);
t = new Thread(rpcServer);
t.start();
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java
index 73caf770512..987dd8a6c4d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java
@@ -14,7 +14,9 @@ import com.yahoo.vespa.config.server.host.HostRegistry;
import com.yahoo.vespa.config.server.http.SessionHandlerTest;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.time.Duration;
@@ -34,6 +36,9 @@ public class LocalSessionRepoTest extends TestWithCurator {
private ManualClock clock;
private static final TenantName tenantName = TenantName.defaultName();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Before
public void setupSessions() throws Exception {
setupSessions(tenantName, true);
@@ -41,7 +46,7 @@ public class LocalSessionRepoTest extends TestWithCurator {
private void setupSessions(TenantName tenantName, boolean createInitialSessions) throws Exception {
GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder().curator(curator).build();
- TenantFileSystemDirs tenantFileSystemDirs = TenantFileSystemDirs.createTestDirs(tenantName);
+ TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(temporaryFolder.newFolder(), tenantName);
if (createInitialSessions) {
IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.sessionsPath(), "1"));
IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.sessionsPath(), "2"));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java
index bd5cfdf3a0e..316c439a3cd 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.config.server.session;
import com.google.common.io.Files;
+import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.config.provision.*;
import com.yahoo.path.Path;
@@ -29,8 +30,7 @@ import static org.hamcrest.core.Is.is;
import static org.junit.Assert.*;
/**
- * @author lulf
- * @since 5.1
+ * @author Ulf Lilleengen
*/
public class LocalSessionTest {
@@ -41,7 +41,7 @@ public class LocalSessionTest {
private SuperModelGenerationCounter superModelGenerationCounter;
@Before
- public void setupTest() throws Exception {
+ public void setupTest() {
curator = new MockCurator();
configCurator = ConfigCurator.create(curator);
superModelGenerationCounter = new SuperModelGenerationCounter(curator);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java
index 9d8b7c5cc00..2c05017d449 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.session;
+import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider;
import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionProvider;
@@ -11,7 +12,12 @@ import java.io.File;
*/
public class MockFileDistributionFactory extends FileDistributionFactory {
- public final MockFileDistributionProvider mockFileDistributionProvider = new MockFileDistributionProvider();
+ public final MockFileDistributionProvider mockFileDistributionProvider;
+
+ public MockFileDistributionFactory(File fileReferencesDir) {
+ super(new ConfigserverConfig(new ConfigserverConfig.Builder()));
+ mockFileDistributionProvider = new MockFileDistributionProvider(fileReferencesDir);
+ }
@Override
public FileDistributionProvider createProvider(File applicationFile) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
index a5f83021bee..7d68e7b6679 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
@@ -96,4 +96,7 @@ public abstract class DocsumField {
*/
public abstract Object convert(Inspector value);
+ /** Returns whether this is the string field type. */
+ boolean isString() { return false; }
+
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
index c517742f0e5..1160ea0a204 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
@@ -228,6 +228,13 @@ public class FastHit extends Hit {
}
@Override
+ public void forEachFieldAsRaw(RawUtf8Consumer consumer) {
+ super.forEachField(consumer);
+ for (SummaryData summaryData : summaries)
+ summaryData.forEachFieldAsRaw(consumer);
+ }
+
+ @Override
public Map<String, Object> fields() {
Map<String, Object> fields = new HashMap<>();
for (Iterator<Map.Entry<String, Object>> i = fieldIterator(); i.hasNext(); ) {
@@ -534,9 +541,28 @@ public class FastHit extends Hit {
void forEachField(BiConsumer<String, Object> consumer) {
data.traverse((ObjectTraverser)(name, value) -> {
- Object convertedValue = type.convert(name, value);
- if ( convertedValue != null && !shadowed(name) && !removed(name)) {
- consumer.accept(name, convertedValue);
+ if (!shadowed(name) && !removed(name)) {
+ Object convertedValue = type.convert(name, value);
+ if (convertedValue != null)
+ consumer.accept(name, convertedValue);
+ }
+ });
+ }
+
+ void forEachFieldAsRaw(RawUtf8Consumer consumer) {
+ data.traverse((ObjectTraverser)(name, value) -> {
+ if (!shadowed(name) && !removed(name)) {
+ DocsumField fieldType = type.getField(name);
+ if (fieldType != null) {
+ if (fieldType.isString()) {
+ byte[] utf8Value = value.asUtf8();
+ consumer.accept(name, utf8Value, 0, utf8Value.length);
+ } else {
+ Object convertedValue = fieldType.convert(value);
+ if (convertedValue != null)
+ consumer.accept(name, convertedValue);
+ }
+ }
}
});
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java
index 54db3b083d7..30e2adab182 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java
@@ -244,7 +244,7 @@ public class FastSearcher extends VespaBackEndSearcher {
if (result.isFilled(summaryClass)) return;
Query query = result.getQuery();
- traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass));
+ traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass));
if (query.properties().getBoolean(dispatchSummaries, true)
&& ! summaryNeedsQuery(query)
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
index 0fa4b7ee342..408cbbbb62d 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
@@ -31,4 +31,6 @@ public class StringField extends DocsumField {
return value.asString("");
}
+ boolean isString() { return true; }
+
}
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
index 6c7018317c3..55c846ccb5b 100644
--- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
+++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
@@ -68,6 +68,7 @@ import java.util.function.LongSupplier;
* JSON renderer for search results.
*
* @author Steinar Knutsen
+ * @author bratseth
*/
// NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc.
public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
@@ -75,18 +76,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug");
private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback");
- private enum RenderDecision {
- YES, NO, DO_NOT_KNOW;
-
- boolean booleanValue() {
- switch (this) {
- case YES: return true;
- case NO: return false;
- default: throw new IllegalStateException();
- }
- }
- }
-
// if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString
private static final String BUCKET_LIMITS = "limits";
private static final String BUCKET_TO = "to";
@@ -133,6 +122,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
private final JsonFactory generatorFactory;
private JsonGenerator generator;
+ private FieldConsumer fieldConsumer;
private Deque<Integer> renderedChildren;
private boolean debugRendering;
private LongSupplier timeSource;
@@ -304,9 +294,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
@Override
public void init() {
super.init();
- generator = null;
- renderedChildren = null;
debugRendering = false;
+ setGenerator(null, debugRendering);
+ renderedChildren = null;
timeSource = System::currentTimeMillis;
stream = null;
}
@@ -314,9 +304,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
@Override
public void beginResponse(OutputStream stream) throws IOException {
beginJsonCallback(stream);
- generator = generatorFactory.createGenerator(stream, JsonEncoding.UTF8);
- renderedChildren = new ArrayDeque<>();
debugRendering = getDebugRendering(getResult().getQuery());
+ setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering);
+ renderedChildren = new ArrayDeque<>();
generator.writeStartObject();
renderTrace(getExecution().trace());
renderTiming();
@@ -473,17 +463,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
return ! (hit instanceof DefaultErrorHit);
}
- private void fieldsStart(MutableBoolean hasFieldsField) throws IOException {
- if (hasFieldsField.get()) return;
- generator.writeObjectFieldStart(FIELDS);
- hasFieldsField.set(true);
- }
-
- private void fieldsEnd(MutableBoolean hasFieldsField) throws IOException {
- if ( ! hasFieldsField.get()) return;
- generator.writeEndObject();
- }
-
private void renderHitContents(Hit hit) throws IOException {
String id = hit.getDisplayId();
if (id != null)
@@ -509,39 +488,14 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
}
private void renderAllFields(Hit hit) throws IOException {
- MutableBoolean hasFieldsField = new MutableBoolean(false);
- renderTotalHitCount(hit, hasFieldsField);
- renderStandardFields(hit, hasFieldsField);
- fieldsEnd(hasFieldsField);
- }
-
- private void renderStandardFields(Hit hit, MutableBoolean hasFieldsField) {
- hit.forEachField((name, value) -> {
- try {
- if (shouldRender(name, value)) {
- fieldsStart(hasFieldsField);
- renderField(name, value);
- }
- }
- catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- });
+ fieldConsumer.startHitFields();
+ renderTotalHitCount(hit);
+ renderStandardFields(hit);
+ fieldConsumer.endHitFields();
}
- private boolean shouldRender(String name, Object value) {
- if (debugRendering) return true;
-
- if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false;
-
- if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false;
-
- // StringFieldValue cannot hold a null, so checking length directly is OK:
- if (value instanceof StringFieldValue && ((StringFieldValue) value).getString().isEmpty()) return false;
-
- if (value instanceof NanNumber) return false;
-
- return true;
+ private void renderStandardFields(Hit hit) {
+ hit.forEachFieldAsRaw(fieldConsumer);
}
private void renderSpecialCasesForGrouping(Hit hit) throws IOException {
@@ -606,96 +560,13 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
return (id instanceof RawBucketId ? Arrays.toString(((RawBucketId) id).getTo()) : id.getTo()).toString();
}
- private void renderTotalHitCount(Hit hit, MutableBoolean hasFieldsField) throws IOException {
+ private void renderTotalHitCount(Hit hit) throws IOException {
if ( ! (getRecursionLevel() == 1 && hit instanceof HitGroup)) return;
- fieldsStart(hasFieldsField);
+ fieldConsumer.ensureFieldsField();
generator.writeNumberField(TOTAL_COUNT, getResult().getTotalHitCount());
- }
-
- private void renderField(String name, Object value) throws IOException {
- generator.writeFieldName(name);
- renderFieldContents(value);
- }
-
- private void renderFieldContents(Object field) throws IOException {
- if (field == null) {
- generator.writeNull();
- } else if (field instanceof Number) {
- renderNumberField((Number) field);
- } else if (field instanceof TreeNode) {
- generator.writeTree((TreeNode) field);
- } else if (field instanceof Tensor) {
- renderTensor(Optional.of((Tensor)field));
- } else if (field instanceof JsonProducer) {
- generator.writeRawValue(((JsonProducer) field).toJson());
- } else if (field instanceof Inspectable) {
- StringBuilder intermediate = new StringBuilder();
- JsonRender.render((Inspectable) field, intermediate, true);
- generator.writeRawValue(intermediate.toString());
- } else if (field instanceof StringFieldValue) {
- // This needs special casing as JsonWriter hides empty strings now
- generator.writeString(((StringFieldValue)field).getString());
- } else if (field instanceof TensorFieldValue) {
- renderTensor(((TensorFieldValue)field).getTensor());
- } else if (field instanceof FieldValue) {
- // the null below is the field which has already been written
- ((FieldValue) field).serialize(null, new JsonWriter(generator));
- } else if (field instanceof JSONArray || field instanceof JSONObject) {
- // org.json returns null if the object would not result in
- // syntactically correct JSON
- String s = field.toString();
- if (s == null) {
- generator.writeNull();
- } else {
- generator.writeRawValue(s);
- }
- } else {
- generator.writeString(field.toString());
- }
- }
-
- private void renderNumberField(Number field) throws IOException {
- if (field instanceof Integer) {
- generator.writeNumber(field.intValue());
- } else if (field instanceof Float) {
- generator.writeNumber(field.floatValue());
- } else if (field instanceof Double) {
- generator.writeNumber(field.doubleValue());
- } else if (field instanceof Long) {
- generator.writeNumber(field.longValue());
- } else if (field instanceof Byte || field instanceof Short) {
- generator.writeNumber(field.intValue());
- } else if (field instanceof BigInteger) {
- generator.writeNumber((BigInteger) field);
- } else if (field instanceof BigDecimal) {
- generator.writeNumber((BigDecimal) field);
- } else {
- generator.writeNumber(field.doubleValue());
- }
- }
-
- private void renderTensor(Optional<Tensor> tensor) throws IOException {
- generator.writeStartObject();
- generator.writeArrayFieldStart("cells");
- if (tensor.isPresent()) {
- for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) {
- Tensor.Cell cell = i.next();
-
- generator.writeStartObject();
-
- generator.writeObjectFieldStart("address");
- for (int d = 0; d < cell.getKey().size(); d++)
- generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d));
- generator.writeEndObject();
-
- generator.writeObjectField("value", cell.getValue());
-
- generator.writeEndObject();
- }
- }
- generator.writeEndArray();
- generator.writeEndObject();
+ // alternative for the above two lines:
+ // fieldConsumer.accept(TOTAL_COUNT, getResult().getTotalHitCount());
}
@Override
@@ -774,11 +645,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
return null;
}
- /**
- * Only for testing. Never to be used in any other context.
- */
- void setGenerator(JsonGenerator generator) {
+ private void setGenerator(JsonGenerator generator, boolean debugRendering) {
this.generator = generator;
+ this.fieldConsumer = generator == null ? null : new FieldConsumer(generator, debugRendering);
}
/**
@@ -787,5 +656,170 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
void setTimeSource(LongSupplier timeSource) {
this.timeSource = timeSource;
}
-
+
+ /**
+ * Received callbacks when fields of hits are encountered.
+ * This instance is reused for all hits of a Result since we are in a single-threaded context
+ * and want to limit object creation.
+ */
+ private static class FieldConsumer implements Hit.RawUtf8Consumer {
+
+ private final JsonGenerator generator;
+ private final boolean debugRendering;
+
+ private MutableBoolean hasFieldsField;
+
+ public FieldConsumer(JsonGenerator generator, boolean debugRendering) {
+ this.generator = generator;
+ this.debugRendering = debugRendering;
+ }
+
+ /**
+ * Call before using this for a hit to track whether we
+ * have created the "fields" field of the JSON object
+ */
+ void startHitFields() {
+ this.hasFieldsField = new MutableBoolean(false);
+ }
+
+ /** Call before rendering a field to the generator */
+ void ensureFieldsField() throws IOException {
+ if (hasFieldsField.get()) return;
+ generator.writeObjectFieldStart(FIELDS);
+ hasFieldsField.set(true);
+ }
+
+ /** Call after all fields in a hit to close the "fields" field of the JSON object */
+ void endHitFields() throws IOException {
+ if ( ! hasFieldsField.get()) return;
+ generator.writeEndObject();
+ this.hasFieldsField = null;
+ }
+
+ @Override
+ public void accept(String name, Object value) {
+ try {
+ if (shouldRender(name, value)) {
+ ensureFieldsField();
+ generator.writeFieldName(name);
+ renderFieldContents(value);
+ }
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ public void accept(String name, byte[] utf8Data, int offset, int length) {
+ try {
+ if (shouldRenderUtf8Value(name, length)) {
+ ensureFieldsField();
+ generator.writeFieldName(name);
+ generator.writeUTF8String(utf8Data, offset, length);
+ }
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private boolean shouldRender(String name, Object value) {
+ if (debugRendering) return true;
+ if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false;
+ if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false;
+ // StringFieldValue cannot hold a null, so checking length directly is OK:
+ if (value instanceof StringFieldValue && ((StringFieldValue) value).getString().isEmpty()) return false;
+ if (value instanceof NanNumber) return false;
+ return true;
+ }
+
+ private boolean shouldRenderUtf8Value(String name, int length) {
+ if (debugRendering) return true;
+ if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false;
+ if (length == 0) return false;
+ return true;
+ }
+
+ private void renderFieldContents(Object field) throws IOException {
+ if (field == null) {
+ generator.writeNull();
+ } else if (field instanceof Number) {
+ renderNumberField((Number) field);
+ } else if (field instanceof TreeNode) {
+ generator.writeTree((TreeNode) field);
+ } else if (field instanceof Tensor) {
+ renderTensor(Optional.of((Tensor)field));
+ } else if (field instanceof JsonProducer) {
+ generator.writeRawValue(((JsonProducer) field).toJson());
+ } else if (field instanceof Inspectable) {
+ StringBuilder intermediate = new StringBuilder();
+ JsonRender.render((Inspectable) field, intermediate, true);
+ generator.writeRawValue(intermediate.toString());
+ } else if (field instanceof StringFieldValue) {
+ generator.writeString(((StringFieldValue)field).getString());
+ } else if (field instanceof TensorFieldValue) {
+ renderTensor(((TensorFieldValue)field).getTensor());
+ } else if (field instanceof FieldValue) {
+ // the null below is the field which has already been written
+ ((FieldValue) field).serialize(null, new JsonWriter(generator));
+ } else if (field instanceof JSONArray || field instanceof JSONObject) {
+ // org.json returns null if the object would not result in
+ // syntactically correct JSON
+ String s = field.toString();
+ if (s == null) {
+ generator.writeNull();
+ } else {
+ generator.writeRawValue(s);
+ }
+ } else {
+ generator.writeString(field.toString());
+ }
+ }
+
+ private void renderNumberField(Number field) throws IOException {
+ if (field instanceof Integer) {
+ generator.writeNumber(field.intValue());
+ } else if (field instanceof Float) {
+ generator.writeNumber(field.floatValue());
+ } else if (field instanceof Double) {
+ generator.writeNumber(field.doubleValue());
+ } else if (field instanceof Long) {
+ generator.writeNumber(field.longValue());
+ } else if (field instanceof Byte || field instanceof Short) {
+ generator.writeNumber(field.intValue());
+ } else if (field instanceof BigInteger) {
+ generator.writeNumber((BigInteger) field);
+ } else if (field instanceof BigDecimal) {
+ generator.writeNumber((BigDecimal) field);
+ } else {
+ generator.writeNumber(field.doubleValue());
+ }
+ }
+
+ private void renderTensor(Optional<Tensor> tensor) throws IOException {
+ generator.writeStartObject();
+ generator.writeArrayFieldStart("cells");
+ if (tensor.isPresent()) {
+ for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) {
+ Tensor.Cell cell = i.next();
+
+ generator.writeStartObject();
+
+ generator.writeObjectFieldStart("address");
+ for (int d = 0; d < cell.getKey().size(); d++)
+ generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d));
+ generator.writeEndObject();
+
+ generator.writeObjectField("value", cell.getValue());
+
+ generator.writeEndObject();
+ }
+ }
+ generator.writeEndArray();
+ generator.writeEndObject();
+ }
+
+ }
+
}
diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java
index f68916c8a68..74c31aa33c5 100644
--- a/container-search/src/main/java/com/yahoo/search/result/Hit.java
+++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java
@@ -404,13 +404,25 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi
/**
* Receive a callback on the given object for each field in this hit.
- * This is the most resource efficient way of traversing all the fields of a hit.
+ * This is more efficient than accessing the fields as a map or iterator.
*/
public void forEachField(BiConsumer<String, Object> consumer) {
if (fields == null) return;
fields.forEach(consumer);
}
+ /**
+ * Receive a callback on the given object for each field in this hit,
+ * where the callback will provide raw utf-8 byte data for strings whose data
+ * is already available at this form.
+ * This is the most resource efficient way of traversing all the fields of a hit
+ * in renderers which produces utf-8.
+ */
+ public void forEachFieldAsRaw(RawUtf8Consumer consumer) {
+ if (fields == null) return;
+ fields.forEach(consumer); // No utf-8 fields available in Hit
+ }
+
/** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */
public Map<String, Object> fields() { return getUnmodifiableFieldMap(); }
@@ -800,4 +812,18 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi
return "hit " + getId() + " (relevance " + getRelevance() + ")";
}
+ public interface RawUtf8Consumer extends BiConsumer<String, Object> {
+
+ /**
+ * Called for fields which are available as UTF-8 instead of accept(String, Object).
+ *
+ * @param fieldName the name of the field
+ * @param utf8Data raw utf-8 data. The reciver <b>must not</b> modify this data
+ * @param offset the start index of the data to accept into the utf8Data array
+ * @param length the length of the data to accept into the utf8Data array
+ */
+ void accept(String fieldName, byte[] utf8Data, int offset, int length);
+
+ }
+
}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
index a9eb9c5e6ce..421f70c8d07 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
@@ -141,9 +141,9 @@ public class SlimeSummaryTestCase {
@Test
public void testFieldAccessAPI() {
- DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf);
DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf);
DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf);
+ DocsumDefinitionSet fullDocsum = createDocsumDefinitionSet(summary_cf);
FastHit hit = new FastHit();
Map<String, Object> expected = new HashMap<>();
@@ -261,6 +261,18 @@ public class SlimeSummaryTestCase {
fieldIterator.remove();
expected.remove("integer_field");
assertFields(expected, hit);
+
+ // --- Add full summary
+ assertNull(fullDocsum.lazyDecode("default", fullishSummary(), hit));
+ expected.put("integer_field", 4);
+ expected.put("short_field", (short)2);
+ expected.put("byte_field", (byte)1);
+ expected.put("float_field", 4.5f);
+ expected.put("double_field", 8.75d);
+ expected.put("int64_field", 8L);
+ expected.put("string_field", "string_value");
+ expected.put("longstring_field", "longstring_value");
+ assertFields(expected, hit);
}
@@ -274,11 +286,16 @@ public class SlimeSummaryTestCase {
traversed.put(name, value);
});
assertEquals(expected, traversed);
+ // raw utf8 field traverser
+ Map<String, Object> traversedUtf8 = new HashMap<>();
+ hit.forEachFieldAsRaw(new Utf8FieldTraverser(traversedUtf8));
+ assertEquals(expected, traversedUtf8);
// fieldKeys
int fieldNameIteratorFieldCount = 0;
for (Iterator<String> i = hit.fieldKeys().iterator(); i.hasNext(); ) {
fieldNameIteratorFieldCount++;
- assertTrue(expected.containsKey(i.next()));
+ String name = i.next();
+ assertTrue("Expected field " + name, expected.containsKey(name));
}
assertEquals(expected.size(), fieldNameIteratorFieldCount);
// fieldKeys
@@ -304,7 +321,7 @@ public class SlimeSummaryTestCase {
return encode((slime));
}
- private byte [] timeoutSummary() {
+ private byte[] timeoutSummary() {
Slime slime = new Slime();
slime.setString("Timed out....");
return encode((slime));
@@ -327,6 +344,22 @@ public class SlimeSummaryTestCase {
return encode((slime));
}
+ private byte[] fullishSummary() {
+ Slime slime = new Slime();
+ Cursor docsum = slime.setObject();
+ docsum.setLong("integer_field", 4);
+ docsum.setLong("short_field", 2);
+ docsum.setLong("byte_field", 1);
+ docsum.setDouble("float_field", 4.5);
+ docsum.setDouble("double_field", 8.75);
+ docsum.setLong("int64_field", 8);
+ docsum.setString("string_field", "string_value");
+ //docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8));
+ docsum.setString("longstring_field", "longstring_value");
+ //docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8));
+ return encode((slime));
+ }
+
private byte[] fullSummary(Tensor tensor1, Tensor tensor2) {
Slime slime = new Slime();
Cursor docsum = slime.setObject();
@@ -346,8 +379,10 @@ public class SlimeSummaryTestCase {
field.setLong("foo", 1);
field.setLong("bar", 2);
}
- docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1));
- docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2));
+ if (tensor1 != null)
+ docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1));
+ if (tensor2 != null)
+ docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2));
return encode((slime));
}
@@ -371,4 +406,26 @@ public class SlimeSummaryTestCase {
return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig);
}
+ private static class Utf8FieldTraverser implements Hit.RawUtf8Consumer {
+
+ private Map<String, Object> traversed;
+
+ public Utf8FieldTraverser(Map<String, Object> traversed) {
+ this.traversed = traversed;
+ }
+
+ @Override
+ public void accept(String fieldName, byte[] utf8Data, int offset, int length) {
+ traversed.put(fieldName, new String(utf8Data, offset, length, StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public void accept(String name, Object value) {
+ if (name.equals("string_value"))
+ fail("Expected string_value to be received as UTF-8");
+ traversed.put(name, value);
+ }
+
+ }
+
}
diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
index 4a0075c0cf1..bf56ad19f44 100644
--- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
@@ -101,7 +101,7 @@ public class JsonRendererTestCase {
}
@Test
- public void testDocumentId() throws IOException, InterruptedException, ExecutionException, JSONException {
+ public void testDocumentId() throws IOException, InterruptedException, ExecutionException {
String expected = "{\n"
+ " \"root\": {\n"
+ " \"children\": [\n"
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
index a2b420c8090..b15acd726d7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.api.integration.configserver;
-import com.fasterxml.jackson.databind.JsonNode;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
@@ -38,8 +37,6 @@ public interface ConfigServer {
void deactivate(DeploymentId applicationInstance) throws NoInstanceException;
- JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds);
-
ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region);
Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 8fb37ba2a15..b65d3bc0849 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller;
-import com.fasterxml.jackson.databind.JsonNode;
import com.google.inject.Inject;
import com.yahoo.component.AbstractComponent;
import com.yahoo.component.Version;
@@ -10,7 +9,6 @@ import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.SystemName;
import com.yahoo.vespa.athenz.api.AthenzDomain;
import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
@@ -157,11 +155,6 @@ public class Controller extends AbstractComponent {
return globalRoutingService.getHealthStatus(rotation.name());
}
- // TODO: Model the response properly
- public JsonNode waitForConfigConvergence(DeploymentId deploymentId, long timeout) {
- return configServer.waitForConfigConverge(deploymentId, timeout);
- }
-
public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName,
String environment, String region) {
return configServer.getApplicationView(tenantName, applicationName, instanceName, environment, region);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
index 516cd52d710..ac0d08f5105 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
@@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
import com.yahoo.yolean.Exceptions;
import java.time.Duration;
-import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
@@ -38,12 +37,8 @@ public class SystemUpgrader extends Maintainer {
if (!target.isPresent()) {
return;
}
- // TODO: Change to SystemApplication.all() once host applications support upgrade
- try {
- deploy(Arrays.asList(SystemApplication.configServer, SystemApplication.zone), target.get());
- } catch (Exception e) {
- log.log(Level.WARNING, "Failed to upgrade system. Retrying in " + maintenanceInterval(), e);
- }
+
+ deploy(SystemApplication.all(), target.get());
}
/** Deploy a list of system applications until they converge on the given version */
@@ -51,14 +46,14 @@ public class SystemUpgrader extends Maintainer {
for (List<ZoneId> zones : controller().zoneRegistry().upgradePolicy().asList()) {
boolean converged = true;
for (ZoneId zone : zones) {
- for (SystemApplication application : applications) {
- boolean dependenciesConverged = application.dependencies().stream()
- .filter(applications::contains) // TODO: Remove when all() is used.
- .allMatch(dependency -> currentVersion(zone, dependency.id()).equals(target));
- if (dependenciesConverged) {
- deploy(target, application, zone);
- }
- converged &= currentVersion(zone, application.id()).equals(target);
+ try {
+ converged &= deployInZone(zone, applications, target);
+ } catch (UnreachableNodeRepositoryException e) {
+ converged = false;
+ log.log(Level.WARNING, e.getMessage() + ". Continuing to next parallel deployed zone");
+ } catch (Exception e) {
+ converged = false;
+ log.log(Level.WARNING, "Failed to upgrade " + zone + ". Continuing to next parallel deployed zone", e);
}
}
if (!converged) {
@@ -67,6 +62,18 @@ public class SystemUpgrader extends Maintainer {
}
}
+ /** @return true if all applications have converged to the target version in the zone */
+ private boolean deployInZone(ZoneId zone, List<SystemApplication> applications, Version target) {
+ boolean converged = true;
+ for (SystemApplication application : applications) {
+ if (convergedOn(target, application.dependencies(), zone)) {
+ deploy(target, application, zone);
+ }
+ converged &= convergedOn(target, application, zone);
+ }
+ return converged;
+ }
+
/** Deploy application on given version idempotently */
private void deploy(Version target, SystemApplication application, ZoneId zone) {
if (!wantedVersion(zone, application.id(), target).equals(target)) {
@@ -75,12 +82,20 @@ public class SystemUpgrader extends Maintainer {
}
}
+ private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneId zone) {
+ return applications.stream().allMatch(application -> convergedOn(target, application, zone));
+ }
+
+ private boolean convergedOn(Version target, SystemApplication application, ZoneId zone) {
+ return currentVersion(zone, application.id(), target).equals(target);
+ }
+
private Version wantedVersion(ZoneId zone, ApplicationId application, Version defaultVersion) {
return minVersion(zone, application, Node::wantedVersion).orElse(defaultVersion);
}
- private Version currentVersion(ZoneId zone, ApplicationId application) {
- return minVersion(zone, application, Node::currentVersion).orElse(Version.emptyVersion);
+ private Version currentVersion(ZoneId zone, ApplicationId application, Version defaultVersion) {
+ return minVersion(zone, application, Node::currentVersion).orElse(defaultVersion);
}
private Optional<Version> minVersion(ZoneId zone, ApplicationId application, Function<Node, Version> versionField) {
@@ -92,9 +107,8 @@ public class SystemUpgrader extends Maintainer {
.map(versionField)
.min(Comparator.naturalOrder());
} catch (Exception e) {
- log.log(Level.WARNING, String.format("Failed to get version for %s in %s: %s", application, zone,
- Exceptions.toMessageString(e)));
- return Optional.empty();
+ throw new UnreachableNodeRepositoryException(String.format("Failed to get version for %s in %s: %s",
+ application, zone, Exceptions.toMessageString(e)));
}
}
@@ -105,4 +119,9 @@ public class SystemUpgrader extends Maintainer {
.map(VespaVersion::versionNumber);
}
+ private class UnreachableNodeRepositoryException extends RuntimeException {
+ private UnreachableNodeRepositoryException(String reason) {
+ super(reason);
+ }
+ }
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index d9848577f0a..10088ba3fea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -168,7 +168,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request);
if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request);
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
- if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/converge")) return waitForConvergence(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request);
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
@@ -579,12 +578,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
return new SlimeJsonResponse(slime);
}
- private HttpResponse waitForConvergence(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
- return new JacksonJsonResponse(controller.waitForConfigConvergence(new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
- ZoneId.from(environment, region)),
- asLong(request.getProperty("timeout"), 1000)));
- }
-
private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region);
ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java
deleted file mode 100644
index cfd6feccf01..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.restapi.application;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.yahoo.container.jdisc.HttpResponse;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * @author bratseth
- */
-public class JacksonJsonResponse extends HttpResponse {
-
- private final JsonNode node;
-
- public JacksonJsonResponse(JsonNode node) {
- super(200);
- this.node = node;
- }
-
- @Override
- public void render(OutputStream stream) throws IOException {
- new ObjectMapper().writeValue(stream, node);
- }
-
- @Override
- public String getContentType() { return "application/json"; }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java
index 99b913709df..ac9afd9752a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java
@@ -1,9 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.inject.Inject;
import com.yahoo.component.AbstractComponent;
import com.yahoo.component.Version;
@@ -186,13 +183,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
applications.remove(deployment.applicationId());
}
- @Override
- public JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds) {
- ObjectNode root = new ObjectNode(JsonNodeFactory.instance);
- root.put("generation", 1);
- return root;
- }
-
// Returns a canned example response
@Override
public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
index 1ec64f8d478..4b563ed203d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
@@ -121,7 +121,6 @@ public class SystemUpgraderTest {
}
@Test
- @Ignore // TODO: Unignore once host applications support upgrade
public void upgrade_system_containing_host_applications() {
tester.controllerTester().zoneRegistry().setUpgradePolicy(
UpgradePolicy.create()
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 407263a7973..8d734ec549c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.TenantName;
-import com.yahoo.io.IOUtils;
import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
@@ -294,14 +293,12 @@ public class ApplicationApiTest extends ControllerContainerTest {
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", POST)
.screwdriverIdentity(SCREWDRIVER_ID),
"Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default");
+
// POST a 'restart application' command with a host filter (other filters not supported yet)
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1", POST)
.screwdriverIdentity(SCREWDRIVER_ID),
"Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default");
- // GET (wait for) convergence
- tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", GET)
- .userIdentity(USER_ID),
- new File("convergence.json"));
+
// GET services
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", GET)
.userIdentity(USER_ID),
@@ -319,11 +316,13 @@ public class ApplicationApiTest extends ControllerContainerTest {
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", DELETE)
.userIdentity(USER_ID),
"Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default");
+
// DELETE (deactivate) a deployment - prod
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE)
.screwdriverIdentity(SCREWDRIVER_ID),
"Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default");
+
// DELETE (deactivate) a deployment is idempotent
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE)
.screwdriverIdentity(SCREWDRIVER_ID),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json
deleted file mode 100644
index acfb67b702b..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
- "generation": 1
-} \ No newline at end of file
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp
index b6aca1ec8be..97dd916bc60 100644
--- a/document/src/tests/documentupdatetestcase.cpp
+++ b/document/src/tests/documentupdatetestcase.cpp
@@ -200,8 +200,6 @@ DocumentUpdateTest::testSimpleUsage() {
ByteBuffer::UP docBuf = serialize42(docUpdate);
docBuf->flip();
DocumentUpdate::UP docUpdateCopy(DocumentUpdate::create42(repo, *docBuf));
- CPPUNIT_ASSERT(!docUpdate.affectsDocumentBody());
- CPPUNIT_ASSERT(!docUpdateCopy->affectsDocumentBody());
// Create a test document
Document doc(*docType, DocumentId("doc::testdoc"));
@@ -252,7 +250,6 @@ DocumentUpdateTest::testSimpleUsage() {
updated.getValue("intarr").release()));
CPPUNIT_ASSERT_EQUAL(size_t(3), val->size());
CPPUNIT_ASSERT_EQUAL(4, (*val)[2].getAsInt());
- CPPUNIT_ASSERT(upd.affectsDocumentBody());
}
{
Document updated(doc);
@@ -593,24 +590,21 @@ void DocumentUpdateTest::testReadSerializedFile()
DocumentUpdate& upd(*updp);
const DocumentType *type = repo.getDocumentType("serializetest");
- CPPUNIT_ASSERT_EQUAL(DocumentId(DocIdString("update", "test")),
- upd.getId());
+ CPPUNIT_ASSERT_EQUAL(DocumentId(DocIdString("update", "test")), upd.getId());
CPPUNIT_ASSERT_EQUAL(*type, upd.getType());
// Verify assign value update.
- FieldUpdate serField = upd[0];
+ FieldUpdate serField = upd.getUpdates()[0];
CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("intfield").getId());
const ValueUpdate* serValue = &serField[0];
CPPUNIT_ASSERT_EQUAL(serValue->getType(), ValueUpdate::Assign);
- const AssignValueUpdate* assign(
- static_cast<const AssignValueUpdate*>(serValue));
- CPPUNIT_ASSERT_EQUAL(IntFieldValue(4),
- static_cast<const IntFieldValue&>(assign->getValue()));
+ const AssignValueUpdate* assign(static_cast<const AssignValueUpdate*>(serValue));
+ CPPUNIT_ASSERT_EQUAL(IntFieldValue(4), static_cast<const IntFieldValue&>(assign->getValue()));
// Verify clear field update.
- serField = upd[1];
+ serField = upd.getUpdates()[1];
CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("floatfield").getId());
serValue = &serField[0];
@@ -618,7 +612,7 @@ void DocumentUpdateTest::testReadSerializedFile()
CPPUNIT_ASSERT(serValue->inherits(ClearValueUpdate::classId));
// Verify add value update.
- serField = upd[2];
+ serField = upd.getUpdates()[2];
CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("arrayoffloatfield").getId());
serValue = &serField[0];
@@ -765,14 +759,11 @@ DocumentUpdateTest::testUpdateArrayEmptyParamValue()
TestDocMan docMan;
Document::UP doc(docMan.createDocument());
const Field &field(doc->getType().getField("tags"));
- CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0,
- doc->getValue(field).get());
+ CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get());
// Assign array field with no array values = empty array.
DocumentUpdate update(*doc->getDataType(), doc->getId());
- update.addUpdate(FieldUpdate(field)
- .addUpdate(AssignValueUpdate(
- ArrayFieldValue(field.getDataType()))));
+ update.addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(ArrayFieldValue(field.getDataType()))));
update.applyTo(*doc);
// Verify that the field was set in the document.
@@ -781,10 +772,9 @@ DocumentUpdateTest::testUpdateArrayEmptyParamValue()
CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size());
// Remove array field.
- update.clear();
- update.addUpdate(FieldUpdate(field)
- .addUpdate(ClearValueUpdate()));
- update.applyTo(*doc);
+ DocumentUpdate update2(*doc->getDataType(), doc->getId());
+ update2.addUpdate(FieldUpdate(field).addUpdate(ClearValueUpdate()));
+ update2.applyTo(*doc);
// Verify that the field was cleared in the document.
std::unique_ptr<ArrayFieldValue> fval2(doc->getAs<ArrayFieldValue>(field));
@@ -798,31 +788,25 @@ DocumentUpdateTest::testUpdateWeightedSetEmptyParamValue()
TestDocMan docMan;
Document::UP doc(docMan.createDocument());
const Field &field(doc->getType().getField("stringweightedset"));
- CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0,
- doc->getValue(field).get());
+ CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get());
// Assign weighted set with no items = empty set.
DocumentUpdate update(*doc->getDataType(), doc->getId());
- update.addUpdate(FieldUpdate(field)
- .addUpdate(AssignValueUpdate(
- WeightedSetFieldValue(field.getDataType()))));
+ update.addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(WeightedSetFieldValue(field.getDataType()))));
update.applyTo(*doc);
// Verify that the field was set in the document.
- std::unique_ptr<WeightedSetFieldValue>
- fval1(doc->getAs<WeightedSetFieldValue>(field));
+ auto fval1(doc->getAs<WeightedSetFieldValue>(field));
CPPUNIT_ASSERT(fval1.get());
CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size());
// Remove weighted set field.
- update.clear();
- update.addUpdate(FieldUpdate(field)
- .addUpdate(ClearValueUpdate()));
- update.applyTo(*doc);
+ DocumentUpdate update2(*doc->getDataType(), doc->getId());
+ update2.addUpdate(FieldUpdate(field).addUpdate(ClearValueUpdate()));
+ update2.applyTo(*doc);
// Verify that the field was cleared in the document.
- std::unique_ptr<WeightedSetFieldValue>
- fval2(doc->getAs<WeightedSetFieldValue>(field));
+ auto fval2(doc->getAs<WeightedSetFieldValue>(field));
CPPUNIT_ASSERT(!fval2);
}
@@ -833,8 +817,7 @@ DocumentUpdateTest::testUpdateArrayWrongSubtype()
TestDocMan docMan;
Document::UP doc(docMan.createDocument());
const Field &field(doc->getType().getField("tags"));
- CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0,
- doc->getValue(field).get());
+ CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get());
// Assign int values to string array.
DocumentUpdate update(*doc->getDataType(), doc->getId());
diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp
index 89f9c0dab67..7058af95828 100644
--- a/document/src/tests/fieldpathupdatetestcase.cpp
+++ b/document/src/tests/fieldpathupdatetestcase.cpp
@@ -221,12 +221,10 @@ ByteBuffer::UP serializeHEAD(const DocumentUpdate & update)
void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) {
try{
- bool affectsBody = a.affectsDocumentBody();
ByteBuffer::UP bb(serializeHEAD(a));
bb->flip();
DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, *bb));
- CPPUNIT_ASSERT_EQUAL(affectsBody, b->affectsDocumentBody());
CPPUNIT_ASSERT_EQUAL(size_t(0), bb->getRemaining());
CPPUNIT_ASSERT_EQUAL(a.getId().toString(), b->getId().toString());
CPPUNIT_ASSERT_EQUAL(a.getUpdates().size(), b->getUpdates().size());
@@ -1021,13 +1019,11 @@ FieldPathUpdateTestCase::testAffectsDocumentBody()
// structmap is body field
{
DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
- CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(),
"structmap{janitor}", std::string(), fv4));
static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true);
docUp.addFieldPathUpdate(update1);
- CPPUNIT_ASSERT(docUp.affectsDocumentBody());
}
// strfoo is header field
@@ -1037,7 +1033,6 @@ FieldPathUpdateTestCase::testAffectsDocumentBody()
"strfoo", std::string(), StringFieldValue("helloworld")));
static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true);
docUp.addFieldPathUpdate(update1);
- CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
}
}
@@ -1070,7 +1065,6 @@ FieldPathUpdateTestCase::testSerializeAssign()
val.setValue("rating", IntFieldValue(100));
DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
- CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{ribbit}", "true", val));
static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true);
@@ -1091,7 +1085,6 @@ FieldPathUpdateTestCase::testSerializeAdd()
adds.add(StringFieldValue("george is getting upset!"));
DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
- CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
FieldPathUpdate::CP update1(new AddFieldPathUpdate(*doc->getDataType(), "strarray", std::string(), adds));
docUp.addFieldPathUpdate(update1);
@@ -1106,7 +1099,6 @@ FieldPathUpdateTestCase::testSerializeRemove()
MapFieldValue mfv(doc->getType().getField("structmap").getDataType());
DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo"));
- CPPUNIT_ASSERT(!docUp.affectsDocumentBody());
FieldPathUpdate::CP update1(new RemoveFieldPathUpdate("structmap{ribbit}", std::string()));
docUp.addFieldPathUpdate(update1);
diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.h b/document/src/vespa/document/serialization/vespadocumentserializer.h
index 3b0fe581c9e..818759d35b5 100644
--- a/document/src/vespa/document/serialization/vespadocumentserializer.h
+++ b/document/src/vespa/document/serialization/vespadocumentserializer.h
@@ -76,7 +76,7 @@ private:
void write(const AssignFieldPathUpdate &value);
void write(const RemoveFieldPathUpdate &value);
- void visit(const DocumentUpdate &value) override { write42(value); }
+ void visit(const DocumentUpdate &value) override { writeHEAD(value); }
void visit(const FieldUpdate &value) override { write(value); }
void visit(const RemoveValueUpdate &value) override { write(value); }
void visit(const AddValueUpdate &value) override { write(value); }
diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp
index fa289ab0bfa..a3cc2a95155 100644
--- a/document/src/vespa/document/update/documentupdate.cpp
+++ b/document/src/vespa/document/update/documentupdate.cpp
@@ -21,27 +21,14 @@ using namespace vespalib::xml;
namespace document {
-IMPLEMENT_IDENTIFIABLE(DocumentUpdate, vespalib::Identifiable);
-
// Declare content bits.
static const unsigned char CONTENT_HASTYPE = 0x01;
-typedef std::vector<FieldUpdate> FieldUpdateList;
-typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateList;
-
-DocumentUpdate::DocumentUpdate()
- : _documentId("doc::"),
- _type(DataType::DOCUMENT),
- _updates(),
- _version(Document::getNewestSerializationVersion()),
- _createIfNonExistent(false)
-{
-}
-
DocumentUpdate::DocumentUpdate(const DataType &type, const DocumentId& id)
: _documentId(id),
_type(&type),
_updates(),
+ _fieldPathUpdates(),
_version(Document::getNewestSerializationVersion()),
_createIfNonExistent(false)
{
@@ -72,7 +59,7 @@ DocumentUpdate::DocumentUpdate(const DocumentTypeRepo& repo,
}
}
-DocumentUpdate::~DocumentUpdate() { }
+DocumentUpdate::~DocumentUpdate() = default;
bool
@@ -92,22 +79,6 @@ DocumentUpdate::operator==(const DocumentUpdate& other) const
return true;
}
-bool
-DocumentUpdate::affectsDocumentBody() const
-{
- for(const auto & update : _updates) {
- if (!update.getField().isHeaderField()) {
- return true;
- }
- }
- for (const auto & update : _fieldPathUpdates) {
- if (update->affectsDocumentBody(*_type)) {
- return true;
- }
- }
- return false;
-}
-
const DocumentType&
DocumentUpdate::getType() const {
return static_cast<const DocumentType &> (*_type);
@@ -125,11 +96,6 @@ DocumentUpdate::addFieldPathUpdate(const FieldPathUpdate::CP& update) {
return *this;
}
-DocumentUpdate*
-DocumentUpdate::clone() const {
- return new DocumentUpdate(*this);
-}
-
void
DocumentUpdate::print(std::ostream& out, bool verbose,
const std::string& indent) const
@@ -183,13 +149,6 @@ DocumentUpdate::applyTo(Document& doc) const
}
void
-DocumentUpdate::serialize42(nbostream &stream) const
-{
- VespaDocumentSerializer serializer(stream);
- serializer.write42(*this);
-}
-
-void
DocumentUpdate::serializeHEAD(nbostream &stream) const
{
VespaDocumentSerializer serializer(stream);
@@ -239,15 +198,13 @@ namespace {
DocumentUpdate::UP
DocumentUpdate::create42(const DocumentTypeRepo& repo, ByteBuffer& buffer)
{
- return std::make_unique<DocumentUpdate>(repo, buffer,
- SerializeVersion::SERIALIZE_42);
+ return std::make_unique<DocumentUpdate>(repo, buffer, SerializeVersion::SERIALIZE_42);
}
DocumentUpdate::UP
DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer)
{
- return std::make_unique<DocumentUpdate>(repo, buffer,
- SerializeVersion::SERIALIZE_HEAD);
+ return std::make_unique<DocumentUpdate>(repo, buffer, SerializeVersion::SERIALIZE_HEAD);
}
void
@@ -333,13 +290,6 @@ DocumentUpdate::deserializeFlags(int sizeAndFlags)
}
void
-DocumentUpdate::onDeserialize42(const DocumentTypeRepo &repo,
- ByteBuffer& buffer)
-{
- deserialize42(repo, buffer);
-}
-
-void
DocumentUpdate::printXml(XmlOutputStream& xos) const
{
xos << XmlTag("document")
diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h
index e6c39278ee2..a7f9d13d693 100644
--- a/document/src/vespa/document/update/documentupdate.h
+++ b/document/src/vespa/document/update/documentupdate.h
@@ -41,15 +41,9 @@ class Document;
* path updates was added, and a new serialization format was
* introduced while keeping the old one.
*/
-class DocumentUpdate : public vespalib::Identifiable,
- public Printable,
- public XmlSerializable
+class DocumentUpdate final : public Printable, public XmlSerializable
{
-public:
- typedef std::unique_ptr<DocumentUpdate> UP;
- typedef std::shared_ptr<DocumentUpdate> SP;
- typedef std::vector<FieldUpdate> FieldUpdateV;
- typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateV;
+private:
/**
* Enum class containing the legal serialization version for
* document updates. This version is not encoded in the serialized
@@ -59,6 +53,11 @@ public:
SERIALIZE_42, // old style format, before vespa 5.0
SERIALIZE_HEAD // new style format, since vespa 5.0
};
+public:
+ typedef std::unique_ptr<DocumentUpdate> UP;
+ typedef std::shared_ptr<DocumentUpdate> SP;
+ typedef std::vector<FieldUpdate> FieldUpdateV;
+ typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateV;
/**
* Create old style document update, no support for field path updates.
@@ -69,7 +68,16 @@ public:
* Create new style document update, possibly with field path updates.
*/
static DocumentUpdate::UP createHEAD(const DocumentTypeRepo&, ByteBuffer&);
-
+
+ /**
+ * Create a document update from a byte buffer containing a serialized
+ * document update. Public to allow useage in std::make_unique/shared.
+ *
+ * @param repo Document type repo used to find proper document type
+ * @param buffer The buffer containing the serialized document update
+ * @param serializeVersion Selector between serialization formats.
+ */
+ DocumentUpdate(const DocumentTypeRepo &repo, ByteBuffer &buffer, SerializeVersion serializeVersion);
/**
* The document type is not strictly needed, as we know this at applyTo()
* time, but search does not use applyTo() code to do the update, and don't
@@ -82,18 +90,9 @@ public:
*/
DocumentUpdate(const DataType &type, const DocumentId& id);
- /**
- * Create a document update from a byte buffer containing a serialized
- * document update.
- *
- * @param repo Document type repo used to find proper document type
- * @param buffer The buffer containing the serialized document update
- * @param serializeVersion Selector between serialization formats.
- */
- DocumentUpdate(const DocumentTypeRepo &repo, ByteBuffer &buffer,
- SerializeVersion serializeVersion);
-
- ~DocumentUpdate();
+ DocumentUpdate(const DocumentUpdate &) = delete;
+ DocumentUpdate & operator = (const DocumentUpdate &) = delete;
+ ~DocumentUpdate() override;
bool operator==(const DocumentUpdate&) const;
bool operator!=(const DocumentUpdate & rhs) const { return ! (*this == rhs); }
@@ -107,11 +106,6 @@ public:
* type as this.
*/
void applyTo(Document& doc) const;
-
- void clear() { _updates.clear(); }
- size_t size() const { return _updates.size(); }
- const FieldUpdate& operator[](int index) const { return _updates[index]; }
- FieldUpdate& operator[](int index) { return _updates[index]; }
/**
* Add a field update to this document update.
@@ -131,26 +125,15 @@ public:
/** @return The list of fieldpath updates. */
const FieldPathUpdateV & getFieldPathUpdates() const { return _fieldPathUpdates; }
- bool affectsDocumentBody() const;
-
/** @return The type of document this update is for. */
const DocumentType& getType() const;
void print(std::ostream& out, bool verbose, const std::string& indent) const override;
- void deserialize42(const DocumentTypeRepo&, ByteBuffer&);
- void deserializeHEAD(const DocumentTypeRepo&, ByteBuffer&);
-
- // Deserializable implementation. Kept as search relies on it currently.
- virtual void onDeserialize42(const DocumentTypeRepo &repo, ByteBuffer&);
-
- void serialize42(vespalib::nbostream &stream) const;
void serializeHEAD(vespalib::nbostream &stream) const;
void printXml(XmlOutputStream&) const override;
- virtual DocumentUpdate* clone() const;
-
/**
* Sets whether this update should create the document it updates if that document does not exist.
* In this case an empty document is created before the update is applied.
@@ -169,7 +152,6 @@ public:
int serializeFlags(int size_) const;
int16_t getVersion() const { return _version; }
- DECLARE_IDENTIFIABLE(DocumentUpdate);
private:
DocumentId _documentId; // The ID of the document to update.
const DataType *_type; // The type of document this update is for.
@@ -178,15 +160,9 @@ private:
int16_t _version; // Serialization version
bool _createIfNonExistent;
- /**
- * This function exist because search relies on deserialization through
- * creating object through empty constructor and calling deserialize.
- *
- * It is hidden to prevent accidental other usage.
- */
- DocumentUpdate();
-
int deserializeFlags(int sizeAndFlags);
+ void deserialize42(const DocumentTypeRepo&, ByteBuffer&);
+ void deserializeHEAD(const DocumentTypeRepo&, ByteBuffer&);
};
} // document
diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp
index 7265329d4d5..239ffff2fef 100644
--- a/document/src/vespa/document/update/fieldpathupdate.cpp
+++ b/document/src/vespa/document/update/fieldpathupdate.cpp
@@ -45,7 +45,7 @@ FieldPathUpdate::FieldPathUpdate(stringref fieldPath, stringref whereClause) :
_originalWhereClause(whereClause)
{ }
-FieldPathUpdate::~FieldPathUpdate() { }
+FieldPathUpdate::~FieldPathUpdate() = default;
bool
FieldPathUpdate::operator==(const FieldPathUpdate& other) const
@@ -76,16 +76,6 @@ FieldPathUpdate::applyTo(Document& doc) const
}
}
-bool
-FieldPathUpdate::affectsDocumentBody(const DataType & type) const
-{
- FieldPath path;
- type.buildFieldPath(path, _originalFieldPath);
- if (path.empty() || !path[0].hasField()) return false;
- const Field& field = path[0].getFieldRef();
- return !field.isHeaderField();
-}
-
void
FieldPathUpdate::print(std::ostream& out, bool, const std::string& indent) const
{
diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h
index c120c8d3979..6629aa74aee 100644
--- a/document/src/vespa/document/update/fieldpathupdate.h
+++ b/document/src/vespa/document/update/fieldpathupdate.h
@@ -64,9 +64,6 @@ public:
*/
void checkCompatibility(const FieldValue& fv, const DataType & type) const;
- /** @return Whether or not the first field path element is a body field */
- bool affectsDocumentBody(const DataType & type) const;
-
void print(std::ostream& out, bool verbose, const std::string& indent) const override;
DECLARE_IDENTIFIABLE_ABSTRACT(FieldPathUpdate);
diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp
index 57396da15ac..277d6467ae8 100644
--- a/document/src/vespa/document/update/fieldupdate.cpp
+++ b/document/src/vespa/document/update/fieldupdate.cpp
@@ -16,21 +16,32 @@ FieldUpdate::FieldUpdate(const Field& field)
{
}
-// Construct a field update by deserialization.
-FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo,
- const DocumentType& type,
- ByteBuffer& buffer,
- int16_t version)
+namespace {
+
+int readInt(ByteBuffer & buffer) {
+ int tmp;
+ buffer.getIntNetwork(tmp);
+ return tmp;
+}
+
+}
+
+FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo, const DocumentType& type, ByteBuffer& buffer, int16_t version)
: Printable(),
- _field(),
+ _field(type.getField(readInt(buffer))),
_updates()
{
- deserialize(repo, type, buffer, version);
+ int numUpdates = readInt(buffer);
+ _updates.reserve(numUpdates);
+ const DataType& dataType = _field.getDataType();
+ for(int i(0); i < numUpdates; i++) {
+ _updates.emplace_back(ValueUpdate::createInstance(repo, dataType, buffer, version).release());
+ }
}
FieldUpdate::FieldUpdate(const FieldUpdate &) = default;
FieldUpdate & FieldUpdate::operator = (const FieldUpdate &) = default;
-FieldUpdate::~FieldUpdate() {}
+FieldUpdate::~FieldUpdate() = default;
bool
FieldUpdate::operator==(const FieldUpdate& other) const
@@ -77,8 +88,7 @@ FieldUpdate::applyTo(Document& doc) const
// Print this field update as a human readable string.
void
-FieldUpdate::print(std::ostream& out, bool verbose,
- const std::string& indent) const
+FieldUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const
{
out << "FieldUpdate(" << _field.toString(verbose);
for(const auto & update : _updates) {
@@ -93,8 +103,8 @@ FieldUpdate::print(std::ostream& out, bool verbose,
// Deserialize this field update from the given buffer.
void
-FieldUpdate::deserialize(const DocumentTypeRepo& repo,
- const DocumentType& docType, ByteBuffer& buffer, int16_t version)
+FieldUpdate::deserialize(const DocumentTypeRepo& repo, const DocumentType& docType,
+ ByteBuffer& buffer, int16_t version)
{
int fieldId;
buffer.getIntNetwork(fieldId);
diff --git a/document/src/vespa/document/update/fieldupdate.h b/document/src/vespa/document/update/fieldupdate.h
index 3f7b3dde3f7..569fb21fe1c 100644
--- a/document/src/vespa/document/update/fieldupdate.h
+++ b/document/src/vespa/document/update/fieldupdate.h
@@ -49,7 +49,7 @@ public:
* @param serializationVersion The serialization version the update was serialized with.
*/
FieldUpdate(const DocumentTypeRepo& repo, const DocumentType& type,
- ByteBuffer& buffer, int16_t serializationVersion);
+ ByteBuffer& buffer, int16_t version);
bool operator==(const FieldUpdate&) const;
bool operator!=(const FieldUpdate & rhs) const { return ! (*this == rhs); }
diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp
index 0346cadbf9b..48728ff6057 100644
--- a/documentapi/src/tests/messages/messages50test.cpp
+++ b/documentapi/src/tests/messages/messages50test.cpp
@@ -7,6 +7,7 @@
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/document/update/fieldpathupdates.h>
#include <vespa/documentapi/documentapi.h>
+#include <vespa/document/bucket/fixed_bucket_spaces.h>
using document::DataType;
using document::DocumentTypeRepo;
@@ -240,6 +241,8 @@ Messages50Test::testRemoveLocationMessage()
if (EXPECT_TRUE(obj.get() != NULL)) {
RemoveLocationMessage &ref = static_cast<RemoveLocationMessage&>(*obj);
EXPECT_EQUAL(string("id.group == \"mygroup\""), ref.getDocumentSelection());
+ // FIXME add to wire format, currently hardcoded.
+ EXPECT_EQUAL(string(document::FixedBucketSpaces::default_space_name()), ref.getBucketSpace());
}
}
}
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
index b5a4a8306b1..26d85b57522 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
@@ -32,7 +32,7 @@ RoutableFactories50::DocumentMessageFactory::decode(document::ByteBuffer &in, co
uint32_t loadClass = decodeInt(in);
DocumentMessage::UP msg = doDecode(in);
- if (msg.get() != NULL) {
+ if (msg) {
msg->setPriority((Priority::Value)pri);
msg->setLoadType(loadTypes[loadClass]);
}
@@ -54,7 +54,7 @@ RoutableFactories50::DocumentReplyFactory::decode(document::ByteBuffer &in, cons
uint8_t pri;
in.getByte(pri);
DocumentReply::UP reply = doDecode(in);
- if (reply.get() != NULL) {
+ if (reply) {
reply->setPriority((Priority::Value)pri);
}
return mbus::Routable::UP(reply.release());
@@ -72,22 +72,17 @@ RoutableFactories50::BatchDocumentUpdateMessageFactory::doDecode(document::ByteB
uint64_t userId = (uint64_t)decodeLong(buf);
string group = decodeString(buf);
- BatchDocumentUpdateMessage* msg;
- if (group.length()) {
- msg = new BatchDocumentUpdateMessage(group);
- } else {
- msg = new BatchDocumentUpdateMessage(userId);
- }
- DocumentMessage::UP retVal(msg);
+ auto msg = (group.length())
+ ? std::make_unique<BatchDocumentUpdateMessage>(group)
+ : std::make_unique<BatchDocumentUpdateMessage>(userId);
uint32_t len = decodeInt(buf);
for (uint32_t i = 0; i < len; i++) {
- document::DocumentUpdate::SP upd;
- upd.reset(document::DocumentUpdate::createHEAD(_repo, buf).release());
+ document::DocumentUpdate::SP upd = document::DocumentUpdate::createHEAD(_repo, buf);
msg->addUpdate(upd);
}
- return retVal;
+ return msg;
}
bool
@@ -111,14 +106,14 @@ RoutableFactories50::BatchDocumentUpdateMessageFactory::doEncode(const DocumentM
DocumentReply::UP
RoutableFactories50::BatchDocumentUpdateReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- BatchDocumentUpdateReply* reply = new BatchDocumentUpdateReply();
+ auto reply = std::make_unique<BatchDocumentUpdateReply>();
reply->setHighestModificationTimestamp(decodeLong(buf));
std::vector<bool>& notFound = reply->getDocumentsNotFound();
notFound.resize(decodeInt(buf));
for (std::size_t i = 0; i < notFound.size(); ++i) {
notFound[i] = decodeBoolean(buf);
}
- return DocumentReply::UP(reply);
+ return reply;
}
bool
@@ -126,10 +121,10 @@ RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentRep
{
const BatchDocumentUpdateReply& reply = static_cast<const BatchDocumentUpdateReply&>(obj);
buf.putLong(reply.getHighestModificationTimestamp());
- const std::vector<bool>& notFound = reply.getDocumentsNotFound();
- buf.putInt(notFound.size());
- for (std::size_t i = 0; i < notFound.size(); ++i) {
- buf.putBoolean(notFound[i]);
+ const std::vector<bool>& notFoundV = reply.getDocumentsNotFound();
+ buf.putInt(notFoundV.size());
+ for (bool notFound : notFoundV) {
+ buf.putBoolean(notFound);
}
return true;
}
@@ -137,34 +132,33 @@ RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentRep
DocumentMessage::UP
RoutableFactories50::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new CreateVisitorMessage());
- CreateVisitorMessage &msg = static_cast<CreateVisitorMessage&>(*ret);
+ auto msg = std::make_unique<CreateVisitorMessage>();
- msg.setLibraryName(decodeString(buf));
- msg.setInstanceId(decodeString(buf));
- msg.setControlDestination(decodeString(buf));
- msg.setDataDestination(decodeString(buf));
- msg.setDocumentSelection(decodeString(buf));
- msg.setMaximumPendingReplyCount(decodeInt(buf));
+ msg->setLibraryName(decodeString(buf));
+ msg->setInstanceId(decodeString(buf));
+ msg->setControlDestination(decodeString(buf));
+ msg->setDataDestination(decodeString(buf));
+ msg->setDocumentSelection(decodeString(buf));
+ msg->setMaximumPendingReplyCount(decodeInt(buf));
int32_t len = decodeInt(buf);
for (int32_t i = 0; i < len; i++) {
int64_t val;
buf.getLong(val); // NOT using getLongNetwork
- msg.getBuckets().push_back(document::BucketId(val));
+ msg->getBuckets().push_back(document::BucketId(val));
}
- msg.setFromTimestamp(decodeLong(buf));
- msg.setToTimestamp(decodeLong(buf));
- msg.setVisitRemoves(decodeBoolean(buf));
- msg.setVisitHeadersOnly(decodeBoolean(buf));
- msg.setVisitInconsistentBuckets(decodeBoolean(buf));
- msg.getParameters().deserialize(_repo, buf);
- msg.setVisitorDispatcherVersion(50);
- msg.setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf));
- msg.setMaxBucketsPerVisitor(decodeInt(buf));
+ msg->setFromTimestamp(decodeLong(buf));
+ msg->setToTimestamp(decodeLong(buf));
+ msg->setVisitRemoves(decodeBoolean(buf));
+ msg->setVisitHeadersOnly(decodeBoolean(buf));
+ msg->setVisitInconsistentBuckets(decodeBoolean(buf));
+ msg->getParameters().deserialize(_repo, buf);
+ msg->setVisitorDispatcherVersion(50);
+ msg->setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf));
+ msg->setMaxBucketsPerVisitor(decodeInt(buf));
- return ret;
+ return msg;
}
bool
@@ -180,11 +174,8 @@ RoutableFactories50::CreateVisitorMessageFactory::doEncode(const DocumentMessage
buf.putInt(msg.getMaximumPendingReplyCount());
buf.putInt(msg.getBuckets().size());
- const std::vector<document::BucketId> &buckets = msg.getBuckets();
- for (std::vector<document::BucketId>::const_iterator it = buckets.begin();
- it != buckets.end(); ++it)
- {
- uint64_t val = it->getRawId();
+ for (const auto & bucketId : msg.getBuckets()) {
+ uint64_t val = bucketId.getRawId();
buf.putBytes((const char*)&val, 8);
}
@@ -208,9 +199,8 @@ RoutableFactories50::CreateVisitorMessageFactory::doEncode(const DocumentMessage
DocumentReply::UP
RoutableFactories50::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new CreateVisitorReply(DocumentProtocol::REPLY_CREATEVISITOR));
- CreateVisitorReply &reply = static_cast<CreateVisitorReply&>(*ret);
- reply.setLastBucket(document::BucketId((uint64_t)decodeLong(buf)));
+ auto reply = std::make_unique<CreateVisitorReply>(DocumentProtocol::REPLY_CREATEVISITOR);
+ reply->setLastBucket(document::BucketId((uint64_t)decodeLong(buf)));
vdslib::VisitorStatistics vs;
vs.setBucketsVisited(decodeInt(buf));
vs.setDocumentsVisited(decodeLong(buf));
@@ -219,9 +209,9 @@ RoutableFactories50::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &b
vs.setBytesReturned(decodeLong(buf));
vs.setSecondPassDocumentsReturned(decodeLong(buf));
vs.setSecondPassBytesReturned(decodeLong(buf));
- reply.setVisitorStatistics(vs);
+ reply->setVisitorStatistics(vs);
- return ret;
+ return reply;
}
bool
@@ -242,10 +232,9 @@ RoutableFactories50::CreateVisitorReplyFactory::doEncode(const DocumentReply &ob
DocumentMessage::UP
RoutableFactories50::DestroyVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new DestroyVisitorMessage());
- DestroyVisitorMessage &msg = static_cast<DestroyVisitorMessage&>(*ret);
- msg.setInstanceId(decodeString(buf));
- return ret;
+ auto msg = std::make_unique<DestroyVisitorMessage>();
+ msg->setInstanceId(decodeString(buf));
+ return msg;
}
bool
@@ -257,35 +246,30 @@ RoutableFactories50::DestroyVisitorMessageFactory::doEncode(const DocumentMessag
}
DocumentReply::UP
-RoutableFactories50::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
- return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DESTROYVISITOR));
+ return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DESTROYVISITOR);
}
bool
-RoutableFactories50::DestroyVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::DestroyVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return true;
}
DocumentMessage::UP
RoutableFactories50::DocumentListMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new DocumentListMessage());
- DocumentListMessage &msg = static_cast<DocumentListMessage&>(*ret);
-
- msg.setBucketId(document::BucketId(decodeLong(buf)));
+ auto msg = std::make_unique<DocumentListMessage>();
+ msg->setBucketId(document::BucketId(decodeLong(buf)));
int32_t len = decodeInt(buf);
for (int32_t i = 0; i < len; i++) {
DocumentListMessage::Entry entry(_repo, buf);
- msg.getDocuments().push_back(entry);
+ msg->getDocuments().push_back(entry);
}
- return ret;
+ return msg;
}
bool
@@ -295,40 +279,36 @@ RoutableFactories50::DocumentListMessageFactory::doEncode(const DocumentMessage
buf.putLong(msg.getBucketId().getRawId());
buf.putInt(msg.getDocuments().size());
- for (uint32_t i = 0; i < msg.getDocuments().size(); i++) {
- int len = msg.getDocuments()[i].getSerializedSize();
+ for (const auto & document : msg.getDocuments()) {
+ int len = document.getSerializedSize();
char *tmp = buf.allocate(len);
document::ByteBuffer dbuf(tmp, len);
- msg.getDocuments()[i].serialize(dbuf);
+ document.serialize(dbuf);
}
return true;
}
DocumentReply::UP
-RoutableFactories50::DocumentListReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::DocumentListReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
- return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTLIST));
+ return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTLIST);
}
bool
-RoutableFactories50::DocumentListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::DocumentListReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return true;
}
DocumentMessage::UP
RoutableFactories50::DocumentSummaryMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new DocumentSummaryMessage());
- DocumentSummaryMessage &msg = static_cast<DocumentSummaryMessage&>(*ret);
+ auto msg = std::make_unique<DocumentSummaryMessage>();
- msg.deserialize(buf);
+ msg->deserialize(buf);
- return ret;
+ return msg;
}
bool
@@ -345,34 +325,30 @@ RoutableFactories50::DocumentSummaryMessageFactory::doEncode(const DocumentMessa
}
DocumentReply::UP
-RoutableFactories50::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
- return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTSUMMARY));
+ return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTSUMMARY);
}
bool
-RoutableFactories50::DocumentSummaryReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::DocumentSummaryReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return true;
}
DocumentMessage::UP
RoutableFactories50::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new EmptyBucketsMessage());
- EmptyBucketsMessage &msg = static_cast<EmptyBucketsMessage&>(*ret);
+ auto msg = std::make_unique<EmptyBucketsMessage>();
int32_t len = decodeInt(buf);
std::vector<document::BucketId> buckets(len);
for (int32_t i = 0; i < len; ++i) {
buckets[i] = document::BucketId(decodeLong(buf));
}
- msg.getBucketIds().swap(buckets);
+ msg->getBucketIds().swap(buckets);
- return ret;
+ return msg;
}
bool
@@ -381,35 +357,28 @@ RoutableFactories50::EmptyBucketsMessageFactory::doEncode(const DocumentMessage
const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj);
buf.putInt(msg.getBucketIds().size());
- const std::vector<document::BucketId> &buckets = msg.getBucketIds();
- for (std::vector<document::BucketId>::const_iterator it = buckets.begin();
- it != buckets.end(); ++it)
- {
- buf.putLong(it->getRawId());
+ for (const auto & bucketId : msg.getBucketIds()) {
+ buf.putLong(bucketId.getRawId());
}
return true;
}
DocumentReply::UP
-RoutableFactories50::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
- return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_EMPTYBUCKETS));
+ return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_EMPTYBUCKETS);
}
bool
-RoutableFactories50::EmptyBucketsReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::EmptyBucketsReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return true;
}
-bool RoutableFactories50::GetBucketListMessageFactory::encodeBucketSpace(
- vespalib::stringref bucketSpace,
- vespalib::GrowableByteBuffer& buf) const {
- (void) buf;
+bool RoutableFactories50::GetBucketListMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace,
+ vespalib::GrowableByteBuffer& ) const
+{
return (bucketSpace == FixedBucketSpaces::default_space_name());
}
@@ -437,18 +406,18 @@ RoutableFactories50::GetBucketListMessageFactory::doEncode(const DocumentMessage
DocumentReply::UP
RoutableFactories50::GetBucketListReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new GetBucketListReply());
- GetBucketListReply &reply = static_cast<GetBucketListReply&>(*ret);
+ auto reply = std::make_unique<GetBucketListReply>();
int32_t len = decodeInt(buf);
+ reply->getBuckets().reserve(len);
for (int32_t i = 0; i < len; i++) {
GetBucketListReply::BucketInfo info;
info._bucket = document::BucketId((uint64_t)decodeLong(buf));
info._bucketInformation = decodeString(buf);
- reply.getBuckets().push_back(info);
+ reply->getBuckets().push_back(info);
}
- return ret;
+ return reply;
}
bool
@@ -458,11 +427,9 @@ RoutableFactories50::GetBucketListReplyFactory::doEncode(const DocumentReply &ob
const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets();
buf.putInt(buckets.size());
- for (std::vector<GetBucketListReply::BucketInfo>::const_iterator it = buckets.begin();
- it != buckets.end(); ++it)
- {
- buf.putLong(it->_bucket.getRawId());
- buf.putString(it->_bucketInformation);
+ for (const auto & bucketInfo : buckets) {
+ buf.putLong(bucketInfo._bucket.getRawId());
+ buf.putString(bucketInfo._bucketInformation);
}
return true;
@@ -471,12 +438,11 @@ RoutableFactories50::GetBucketListReplyFactory::doEncode(const DocumentReply &ob
DocumentMessage::UP
RoutableFactories50::GetBucketStateMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new GetBucketStateMessage());
- GetBucketStateMessage &msg = static_cast<GetBucketStateMessage&>(*ret);
+ auto msg = std::make_unique<GetBucketStateMessage>();
- msg.setBucketId(document::BucketId((uint64_t)decodeLong(buf)));
+ msg->setBucketId(document::BucketId((uint64_t)decodeLong(buf)));
- return ret;
+ return msg;
}
bool
@@ -490,16 +456,15 @@ RoutableFactories50::GetBucketStateMessageFactory::doEncode(const DocumentMessag
DocumentReply::UP
RoutableFactories50::GetBucketStateReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new GetBucketStateReply());
- GetBucketStateReply &reply = static_cast<GetBucketStateReply&>(*ret);
+ auto reply = std::make_unique<GetBucketStateReply>();
int32_t len = decodeInt(buf);
+ reply->getBucketState().reserve(len);
for (int32_t i = 0; i < len; i++) {
- DocumentState state(buf);
- reply.getBucketState().push_back(state);
+ reply->getBucketState().emplace_back(buf);
}
- return ret;
+ return reply;
}
bool
@@ -508,11 +473,8 @@ RoutableFactories50::GetBucketStateReplyFactory::doEncode(const DocumentReply &o
const GetBucketStateReply &reply = static_cast<const GetBucketStateReply&>(obj);
buf.putInt(reply.getBucketState().size());
- const std::vector<DocumentState> &state = reply.getBucketState();
- for (std::vector<DocumentState>::const_iterator it = state.begin();
- it != state.end(); ++it)
- {
- it->serialize(buf);
+ for (const auto & state : reply.getBucketState()) {
+ state.serialize(buf);
}
return true;
@@ -521,13 +483,11 @@ RoutableFactories50::GetBucketStateReplyFactory::doEncode(const DocumentReply &o
DocumentMessage::UP
RoutableFactories50::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new GetDocumentMessage());
- GetDocumentMessage &msg = static_cast<GetDocumentMessage&>(*ret);
-
- msg.setDocumentId(decodeDocumentId(buf));
- msg.setFlags(decodeInt(buf));
+ auto msg = std::make_unique<GetDocumentMessage>();
+ msg->setDocumentId(decodeDocumentId(buf));
+ msg->setFlags(decodeInt(buf));
- return ret;
+ return msg;
}
bool
@@ -544,23 +504,22 @@ RoutableFactories50::GetDocumentMessageFactory::doEncode(const DocumentMessage &
DocumentReply::UP
RoutableFactories50::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new GetDocumentReply());
- GetDocumentReply &reply = static_cast<GetDocumentReply&>(*ret);
+ auto reply = std::make_unique<GetDocumentReply>();
bool hasDocument = decodeBoolean(buf);
document::Document * document = nullptr;
if (hasDocument) {
auto doc = std::make_shared<document::Document>(_repo, buf);
document = doc.get();
- reply.setDocument(std::move(doc));
+ reply->setDocument(std::move(doc));
}
int64_t lastModified = decodeLong(buf);
- reply.setLastModified(lastModified);
+ reply->setLastModified(lastModified);
if (hasDocument) {
document->setLastModified(lastModified);
}
- return ret;
+ return reply;
}
bool
@@ -582,12 +541,9 @@ RoutableFactories50::GetDocumentReplyFactory::doEncode(const DocumentReply &obj,
DocumentMessage::UP
RoutableFactories50::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new MapVisitorMessage());
- MapVisitorMessage &msg = static_cast<MapVisitorMessage&>(*ret);
-
- msg.getData().deserialize(_repo, buf);
-
- return ret;
+ auto msg = std::make_unique<MapVisitorMessage>();
+ msg->getData().deserialize(_repo, buf);
+ return msg;
}
bool
@@ -604,17 +560,14 @@ RoutableFactories50::MapVisitorMessageFactory::doEncode(const DocumentMessage &o
}
DocumentReply::UP
-RoutableFactories50::MapVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::MapVisitorReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
- return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MAPVISITOR));
+ return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_MAPVISITOR);
}
bool
-RoutableFactories50::MapVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::MapVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return true;
}
@@ -672,11 +625,10 @@ RoutableFactories50::RemoveDocumentMessageFactory::doEncode(const DocumentMessag
DocumentReply::UP
RoutableFactories50::RemoveDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new RemoveDocumentReply());
- RemoveDocumentReply &reply = static_cast<RemoveDocumentReply&>(*ret);
- reply.setWasFound(decodeBoolean(buf));
- reply.setHighestModificationTimestamp(decodeLong(buf));
- return ret;
+ auto reply = std::make_unique<RemoveDocumentReply>();
+ reply->setWasFound(decodeBoolean(buf));
+ reply->setHighestModificationTimestamp(decodeLong(buf));
+ return reply;
}
bool
@@ -696,7 +648,10 @@ RoutableFactories50::RemoveLocationMessageFactory::doDecode(document::ByteBuffer
document::BucketIdFactory factory;
document::select::Parser parser(_repo, factory);
- return DocumentMessage::UP(new RemoveLocationMessage(factory, parser, selection));
+ auto msg = std::make_unique<RemoveLocationMessage>(factory, parser, selection);
+ // FIXME bucket space not part of wire format, implicitly limiting to only default space for now.
+ msg->setBucketSpace(document::FixedBucketSpaces::default_space_name());
+ return msg;
}
bool
@@ -710,7 +665,7 @@ RoutableFactories50::RemoveLocationMessageFactory::doEncode(const DocumentMessag
DocumentReply::UP
RoutableFactories50::RemoveLocationReplyFactory::doDecode(document::ByteBuffer &) const
{
- return DocumentReply::UP(new DocumentReply(DocumentProtocol::REPLY_REMOVELOCATION));
+ return std::make_unique<DocumentReply>(DocumentProtocol::REPLY_REMOVELOCATION);
}
bool
@@ -722,12 +677,9 @@ RoutableFactories50::RemoveLocationReplyFactory::doEncode(const DocumentReply &,
DocumentMessage::UP
RoutableFactories50::SearchResultMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new SearchResultMessage());
- SearchResultMessage &msg = static_cast<SearchResultMessage&>(*ret);
-
- msg.deserialize(buf);
-
- return ret;
+ auto msg = std::make_unique<SearchResultMessage>();
+ msg->deserialize(buf);
+ return msg;
}
bool
@@ -746,13 +698,11 @@ RoutableFactories50::SearchResultMessageFactory::doEncode(const DocumentMessage
DocumentMessage::UP
RoutableFactories50::QueryResultMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new QueryResultMessage());
- QueryResultMessage &msg = static_cast<QueryResultMessage&>(*ret);
-
- msg.getSearchResult().deserialize(buf);
- msg.getDocumentSummary().deserialize(buf);
+ auto msg = std::make_unique<QueryResultMessage>();
+ msg->getSearchResult().deserialize(buf);
+ msg->getDocumentSummary().deserialize(buf);
- return ret;
+ return msg;
}
bool
@@ -770,39 +720,32 @@ RoutableFactories50::QueryResultMessageFactory::doEncode(const DocumentMessage &
}
DocumentReply::UP
-RoutableFactories50::SearchResultReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::SearchResultReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
- return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_SEARCHRESULT));
+ return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT);
}
bool
-RoutableFactories50::SearchResultReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::SearchResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return true;
}
DocumentReply::UP
-RoutableFactories50::QueryResultReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::QueryResultReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
- return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_QUERYRESULT));
+ return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_QUERYRESULT);
}
bool
-RoutableFactories50::QueryResultReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::QueryResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return true;
}
-bool RoutableFactories50::StatBucketMessageFactory::encodeBucketSpace(
- vespalib::stringref bucketSpace,
- vespalib::GrowableByteBuffer& buf) const {
- (void) buf;
+bool RoutableFactories50::StatBucketMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace,
+ vespalib::GrowableByteBuffer& ) const
+{
return (bucketSpace == FixedBucketSpaces::default_space_name());
}
@@ -813,14 +756,13 @@ string RoutableFactories50::StatBucketMessageFactory::decodeBucketSpace(document
DocumentMessage::UP
RoutableFactories50::StatBucketMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new StatBucketMessage());
- StatBucketMessage &msg = static_cast<StatBucketMessage&>(*ret);
+ auto msg = std::make_unique<StatBucketMessage>();
- msg.setBucketId(document::BucketId(decodeLong(buf)));
- msg.setDocumentSelection(decodeString(buf));
- msg.setBucketSpace(decodeBucketSpace(buf));
+ msg->setBucketId(document::BucketId(decodeLong(buf)));
+ msg->setDocumentSelection(decodeString(buf));
+ msg->setBucketSpace(decodeBucketSpace(buf));
- return ret;
+ return msg;
}
bool
@@ -836,12 +778,9 @@ RoutableFactories50::StatBucketMessageFactory::doEncode(const DocumentMessage &o
DocumentReply::UP
RoutableFactories50::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new StatBucketReply());
- StatBucketReply &reply = static_cast<StatBucketReply&>(*ret);
-
- reply.setResults(decodeString(buf));
-
- return ret;
+ auto reply = std::make_unique<StatBucketReply>();
+ reply->setResults(decodeString(buf));
+ return reply;
}
bool
@@ -853,41 +792,32 @@ RoutableFactories50::StatBucketReplyFactory::doEncode(const DocumentReply &obj,
}
DocumentMessage::UP
-RoutableFactories50::StatDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::StatDocumentMessageFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
return DocumentMessage::UP(); // TODO: remove message type
}
bool
-RoutableFactories50::StatDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::StatDocumentMessageFactory::doEncode(const DocumentMessage &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return false;
}
DocumentReply::UP
-RoutableFactories50::StatDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::StatDocumentReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
return DocumentReply::UP(); // TODO: remove reply type
}
bool
-RoutableFactories50::StatDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::StatDocumentReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return false;
}
void
RoutableFactories50::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const {
- msg.setDocumentUpdate(make_shared<document::DocumentUpdate>
- (_repo, buf,
- document::DocumentUpdate::SerializeVersion::
- SERIALIZE_HEAD));
+ msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, buf));
msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf)));
msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf)));
}
@@ -909,11 +839,10 @@ RoutableFactories50::UpdateDocumentMessageFactory::doEncode(const DocumentMessag
DocumentReply::UP
RoutableFactories50::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new UpdateDocumentReply());
- UpdateDocumentReply &reply = static_cast<UpdateDocumentReply&>(*ret);
- reply.setWasFound(decodeBoolean(buf));
- reply.setHighestModificationTimestamp(decodeLong(buf));
- return ret;
+ auto reply = std::make_unique<UpdateDocumentReply>();
+ reply->setWasFound(decodeBoolean(buf));
+ reply->setHighestModificationTimestamp(decodeLong(buf));
+ return reply;
}
bool
@@ -928,18 +857,18 @@ RoutableFactories50::UpdateDocumentReplyFactory::doEncode(const DocumentReply &o
DocumentMessage::UP
RoutableFactories50::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentMessage::UP ret(new VisitorInfoMessage());
- VisitorInfoMessage &msg = static_cast<VisitorInfoMessage&>(*ret);
+ auto msg = std::make_unique<VisitorInfoMessage>();
int32_t len = decodeInt(buf);
+ msg->getFinishedBuckets().reserve(len);
for (int32_t i = 0; i < len; i++) {
int64_t val;
buf.getLong(val); // NOT using getLongNetwork
- msg.getFinishedBuckets().push_back(document::BucketId(val));
+ msg->getFinishedBuckets().emplace_back(val);
}
- msg.setErrorMessage(decodeString(buf));
+ msg->setErrorMessage(decodeString(buf));
- return ret;
+ return msg;
}
bool
@@ -948,11 +877,8 @@ RoutableFactories50::VisitorInfoMessageFactory::doEncode(const DocumentMessage &
const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj);
buf.putInt(msg.getFinishedBuckets().size());
- const std::vector<document::BucketId> &buckets = msg.getFinishedBuckets();
- for (std::vector<document::BucketId>::const_iterator it = buckets.begin();
- it != buckets.end(); ++it)
- {
- uint64_t val = it->getRawId();
+ for (const auto & bucketId : msg.getFinishedBuckets()) {
+ uint64_t val = bucketId.getRawId();
buf.putBytes((const char*)&val, 8);
}
buf.putString(msg.getErrorMessage());
@@ -961,29 +887,23 @@ RoutableFactories50::VisitorInfoMessageFactory::doEncode(const DocumentMessage &
}
DocumentReply::UP
-RoutableFactories50::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &buf) const
+RoutableFactories50::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &) const
{
- (void)buf;
- return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_VISITORINFO));
+ return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_VISITORINFO);
}
bool
-RoutableFactories50::VisitorInfoReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
+RoutableFactories50::VisitorInfoReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const
{
- (void)obj;
- (void)buf;
return true;
}
DocumentReply::UP
RoutableFactories50::WrongDistributionReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new WrongDistributionReply());
- WrongDistributionReply &reply = static_cast<WrongDistributionReply&>(*ret);
-
- reply.setSystemState(decodeString(buf));
-
- return ret;
+ auto reply = std::make_unique<WrongDistributionReply>();
+ reply->setSystemState(decodeString(buf));
+ return reply;
}
bool
@@ -1013,19 +933,19 @@ RoutableFactories50::FeedMessageFactory::myEncode(const FeedMessage &msg, vespal
DocumentReply::UP
RoutableFactories50::FeedReplyFactory::doDecode(document::ByteBuffer &buf) const
{
- DocumentReply::UP ret(new FeedReply(getType()));
- FeedReply &reply = static_cast<FeedReply&>(*ret);
+ auto reply = std::make_unique<FeedReply>(getType());
- std::vector<FeedAnswer> &answers = reply.getFeedAnswers();
+ std::vector<FeedAnswer> &answers = reply->getFeedAnswers();
int32_t len = decodeInt(buf);
+ answers.reserve(len);
for (int32_t i = 0; i < len; ++i) {
int32_t typeCode = decodeInt(buf);
int32_t wantedIncrement = decodeInt(buf);
string recipient = decodeString(buf);
string moreInfo = decodeString(buf);
- answers.push_back(FeedAnswer(typeCode, wantedIncrement, recipient, moreInfo));
+ answers.emplace_back(typeCode, wantedIncrement, recipient, moreInfo);
}
- return ret;
+ return reply;
}
bool
@@ -1033,14 +953,11 @@ RoutableFactories50::FeedReplyFactory::doEncode(const DocumentReply &obj, vespal
{
const FeedReply &reply = static_cast<const FeedReply&>(obj);
buf.putInt(reply.getFeedAnswers().size());
- const std::vector<FeedAnswer> &answers = reply.getFeedAnswers();
- for (std::vector<FeedAnswer>::const_iterator it = answers.begin();
- it != answers.end(); ++it)
- {
- buf.putInt(it->getAnswerCode());
- buf.putInt(it->getWantedIncrement());
- buf.putString(it->getRecipient());
- buf.putString(it->getMoreInfo());
+ for (const auto & answer : reply.getFeedAnswers()) {
+ buf.putInt(answer.getAnswerCode());
+ buf.putInt(answer.getWantedIncrement());
+ buf.putString(answer.getRecipient());
+ buf.putString(answer.getMoreInfo());
}
return true;
}
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp
index ce83a2cd638..c7f3401d3e1 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp
@@ -17,8 +17,7 @@ RoutableRepository::VersionMap::VersionMap() :
{ }
bool
-RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version,
- IRoutableFactory::SP factory)
+RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version, IRoutableFactory::SP factory)
{
bool ret = _factoryVersions.find(version) != _factoryVersions.end();
_factoryVersions[version] = factory;
@@ -41,7 +40,8 @@ RoutableRepository::VersionMap::getFactory(const vespalib::Version &version) con
return IRoutableFactory::SP();
}
- return std::max_element(candidates.begin(), candidates.end(), [](auto & lhs, auto & rhs) { return lhs.first.compareTo(rhs.first) <= 0; })->second;
+ return std::max_element(candidates.begin(), candidates.end(),
+ [](auto & lhs, auto & rhs) { return lhs.first.compareTo(rhs.first) <= 0; })->second;
}
RoutableRepository::RoutableRepository(const LoadTypeSet& loadTypes) :
@@ -50,7 +50,6 @@ RoutableRepository::RoutableRepository(const LoadTypeSet& loadTypes) :
_cache(),
_loadTypes(loadTypes)
{
- // empty
}
mbus::Routable::UP
@@ -65,13 +64,13 @@ RoutableRepository::decode(const vespalib::Version &version, mbus::BlobRef data)
int type;
in.getIntNetwork(type);
IRoutableFactory::SP factory = getFactory(version, type);
- if (factory.get() == NULL) {
+ if (!factory) {
LOG(error, "No routable factory found for routable type %d (version %s).",
type, version.toString().c_str());
return mbus::Routable::UP();
}
mbus::Routable::UP ret = factory->decode(in, _loadTypes);
- if (ret.get() == NULL) {
+ if (!ret) {
LOG(error, "Routable factory failed to deserialize routable of type %d (version %s).",
type, version.toString().c_str());
@@ -89,7 +88,7 @@ RoutableRepository::encode(const vespalib::Version &version, const mbus::Routabl
uint32_t type = obj.getType();
IRoutableFactory::SP factory = getFactory(version, type);
- if (factory.get() == NULL) {
+ if (!factory) {
LOG(error, "No routable factory found for routable type %d (version %s).",
type, version.toString().c_str());
return mbus::Blob(0);
@@ -130,7 +129,7 @@ RoutableRepository::getFactory(const vespalib::Version &version, uint32_t type)
return IRoutableFactory::SP();
}
IRoutableFactory::SP factory = vit->second.getFactory(version);
- if (factory.get() == NULL) {
+ if (!factory) {
return IRoutableFactory::SP();
}
_cache[cacheKey] = factory;
@@ -141,11 +140,9 @@ uint32_t
RoutableRepository::getRoutableTypes(const vespalib::Version &version, std::vector<uint32_t> &out) const
{
vespalib::LockGuard guard(_lock);
- for (TypeMap::const_iterator it = _factoryTypes.begin();
- it != _factoryTypes.end(); ++it)
- {
- if (it->second.getFactory(version).get() != NULL) {
- out.push_back(it->first);
+ for (const auto & type : _factoryTypes) {
+ if (type.second.getFactory(version)) {
+ out.push_back(type.first);
}
}
return _factoryTypes.size();
diff --git a/fat-model-dependencies/OWNERS b/fat-model-dependencies/OWNERS
new file mode 100644
index 00000000000..d34761f1ba5
--- /dev/null
+++ b/fat-model-dependencies/OWNERS
@@ -0,0 +1,2 @@
+gjoranv
+hmusum
diff --git a/fat-model-dependencies/README b/fat-model-dependencies/README
new file mode 100644
index 00000000000..ba71b189db9
--- /dev/null
+++ b/fat-model-dependencies/README
@@ -0,0 +1,4 @@
+This module contains all dependencies that must be embedded in the config-model-fat bundle.
+This artifact should be depended on by config-model-fat and all amended versions of the
+fat config model. This allows pulling in the same set of dependencies without duplication
+in pom.xml.
diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml
new file mode 100644
index 00000000000..1415ca6e5aa
--- /dev/null
+++ b/fat-model-dependencies/pom.xml
@@ -0,0 +1,223 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+ <artifactId>fat-model-dependencies</artifactId>
+ <packaging>pom</packaging>
+ <version>6-SNAPSHOT</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-model</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <!-- Large, and installed separately as part of Vespa -->
+ <groupId>org.tensorflow</groupId>
+ <artifactId>libtensorflow_jni</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-lib</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>provided-dependencies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>configdefinitions</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-application-package</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>configgen</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-bundle</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>jrt</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-lib</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>simplemetrics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>metrics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-disc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespajlib</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>yolean</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>documentapi</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vdslib</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>messagebus</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>document</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>linguistics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespalog</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>statistics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>messagebus-disc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-messagebus</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>searchlib</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>processing</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>chain</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>docproc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-search</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <!-- OPTIMIZATION: very large (44 MB) and only used for query sorting -->
+ <groupId>com.ibm.icu</groupId>
+ <artifactId>icu4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-search-and-docproc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>logd</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>searchcore</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>storage</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vsm</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>indexinglanguage</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>searchsummary</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.major-version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>jdisc_http_service</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 0c5dd72c968..7f2d1f1eff7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -5,16 +5,19 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yahoo.collections.Pair;
+import com.yahoo.config.provision.NodeType;
import com.yahoo.io.IOUtils;
import com.yahoo.system.ProcessExecuter;
import com.yahoo.vespa.hosted.dockerapi.ContainerName;
import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper;
import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions;
+import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper;
import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider;
import com.yahoo.vespa.hosted.node.admin.component.Environment;
+import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil;
import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger;
import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig;
@@ -46,6 +49,7 @@ public class StorageMaintainer {
private static final ContainerName NODE_ADMIN = new ContainerName("node-admin");
private static final ObjectMapper objectMapper = new ObjectMapper();
+ private final GaugeWrapper numberOfCoredumpsOnHost;
private final CounterWrapper numberOfNodeAdminMaintenanceFails;
private final DockerOperations dockerOperations;
private final ProcessExecuter processExecuter;
@@ -54,7 +58,6 @@ public class StorageMaintainer {
private final Map<ContainerName, MaintenanceThrottler> maintenanceThrottlerByContainerName = new ConcurrentHashMap<>();
-
public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter, MetricReceiverWrapper metricReceiver, Environment environment, Clock clock) {
this.dockerOperations = dockerOperations;
this.processExecuter = processExecuter;
@@ -63,44 +66,98 @@ public class StorageMaintainer {
Dimensions dimensions = new Dimensions.Builder().add("role", "docker").build();
numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.maintenance.fails");
+ numberOfCoredumpsOnHost = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.coredumps");
}
public void writeMetricsConfig(ContainerName containerName, NodeSpec node) {
- final Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode(
- containerName, Paths.get("/etc/yamas-agent/"));
-
- Path vespaCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa");
- SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all")
- .withTag("parentHostname", environment.getParentHostHostname());
+ List<SecretAgentCheckConfig> configs = new ArrayList<>();
+ // host-life
Path hostLifeCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life");
- SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath)
- .withTag("namespace", "Vespa")
+ SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath);
+ configs.add(annotatedCheck(node, hostLifeSchedule));
+
+ // ntp
+ Path ntpCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ntp");
+ SecretAgentCheckConfig ntpSchedule = new SecretAgentCheckConfig("ntp", 60, ntpCheckPath);
+ configs.add(annotatedCheck(node, ntpSchedule));
+
+ // coredumps (except for the done coredumps which is handled by the host)
+ Path coredumpCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps");
+ SecretAgentCheckConfig coredumpSchedule = new SecretAgentCheckConfig("system-coredumps-processing", 300,
+ coredumpCheckPath, "--application", "system-coredumps-processing", "--lastmin",
+ "129600", "--crit", "1", "--coredir", environment.pathInNodeUnderVespaHome("var/crash/processing").toString());
+ configs.add(annotatedCheck(node, coredumpSchedule));
+
+ if (node.getNodeType() != NodeType.config) {
+ // vespa-health
+ Path vespaHealthCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health");
+ SecretAgentCheckConfig vespaHealthSchedule = new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all");
+ configs.add(annotatedCheck(node, vespaHealthSchedule));
+
+ // vespa
+ Path vespaCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa");
+ SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all");
+ configs.add(annotatedCheck(node, vespaSchedule));
+ }
+
+ if (node.getNodeType() == NodeType.config) {
+ // configserver
+ Path configServerCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ymonsb2");
+ SecretAgentCheckConfig configServerSchedule = new SecretAgentCheckConfig("configserver", 60,
+ configServerCheckPath, "-zero", "configserver");
+ configs.add(annotatedCheck(node, configServerSchedule));
+
+ //zkbackupage
+ Path zkbackupCheckPath = environment.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
+ SecretAgentCheckConfig zkbackupSchedule = new SecretAgentCheckConfig("zkbackupage", 300,
+ zkbackupCheckPath, "-f", environment.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(),
+ "-m", "150", "-a", "config-zkbackupage");
+ configs.add(annotatedCheck(node, zkbackupSchedule));
+ }
+
+ if (node.getNodeType() == NodeType.proxy) {
+ //routing-configage
+ Path routingAgeCheckPath = environment.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py");
+ SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60,
+ routingAgeCheckPath, "-f", environment.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(),
+ "-m", "90", "-a", "routing-configage");
+ configs.add(annotatedCheck(node, routingAgeSchedule));
+
+ //ssl-check
+ Path sslCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status");
+ SecretAgentCheckConfig sslSchedule = new SecretAgentCheckConfig("ssl-status", 300,
+ sslCheckPath, "-e", "localhost", "-p", "4443", "-t", "30");
+ configs.add(annotatedCheck(node, sslSchedule));
+ }
+
+ // Write config and restart yamas-agent
+ Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/etc/yamas-agent/"));
+ configs.forEach(s -> IOExceptionUtil.uncheck(() -> s.writeTo(yamasAgentFolder)));
+ final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"};
+ dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent);
+ }
+
+ private SecretAgentCheckConfig annotatedCheck(NodeSpec node, SecretAgentCheckConfig check) {
+ check.withTag("namespace", "Vespa")
.withTag("role", "tenants")
.withTag("flavor", node.getFlavor())
.withTag("canonicalFlavor", node.getCanonicalFlavor())
.withTag("state", node.getState().toString())
.withTag("zone", environment.getZone())
.withTag("parentHostname", environment.getParentHostHostname());
- node.getOwner().ifPresent(owner -> hostLifeSchedule
+ node.getOwner().ifPresent(owner -> check
.withTag("tenantName", owner.getTenant())
.withTag("app", owner.getApplication() + "." + owner.getInstance())
.withTag("applicationName", owner.getApplication())
.withTag("instanceName", owner.getInstance())
.withTag("applicationId", owner.getTenant() + "." + owner.getApplication() + "." + owner.getInstance()));
- node.getMembership().ifPresent(membership -> hostLifeSchedule
+ node.getMembership().ifPresent(membership -> check
.withTag("clustertype", membership.getClusterType())
.withTag("clusterid", membership.getClusterId()));
- node.getVespaVersion().ifPresent(version -> hostLifeSchedule.withTag("vespaVersion", version));
+ node.getVespaVersion().ifPresent(version -> check.withTag("vespaVersion", version));
- try {
- vespaSchedule.writeTo(yamasAgentFolder);
- hostLifeSchedule.writeTo(yamasAgentFolder);
- final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"};
- dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent);
- } catch (IOException e) {
- throw new RuntimeException("Failed to write secret-agent schedules for " + containerName, e);
- }
+ return check;
}
public void writeFilebeatConfig(ContainerName containerName, NodeSpec node) {
@@ -218,6 +275,14 @@ public class StorageMaintainer {
* @param force Set to true to bypass throttling
*/
public void handleCoreDumpsForContainer(ContainerName containerName, NodeSpec node, boolean force) {
+ // Sample number of coredumps on the host
+ try {
+ numberOfCoredumpsOnHost.sample(Files.list(environment.pathInNodeAdminToDoneCoredumps()).count());
+ } catch (IOException e) {
+ // Ignore for now - this is either test or a misconfiguration
+ }
+
+ // Return early if throttled
if (! getMaintenanceThrottlerFor(containerName).shouldHandleCoredumpsNow() && !force) return;
MaintainerExecutor maintainerExecutor = new MaintainerExecutor();
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
index f7e9c3ca1d8..bd75368a0dc 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java
@@ -148,6 +148,7 @@ public class AthenzCredentialsMaintainer {
}
}
+ @SuppressWarnings("deprecation")
private VespaUniqueInstanceId getVespaUniqueInstanceId(NodeSpec nodeSpec) {
NodeSpec.Membership membership = nodeSpec.getMembership().get();
NodeSpec.Owner owner = nodeSpec.getOwner().get();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
index 12a0496ed2e..203f52a7b70 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
@@ -38,6 +38,8 @@ public class CuratorDatabase {
/** Whether we should return data from the cache or always read fro ZooKeeper */
private final boolean useCache;
+ private final Object cacheCreationLock = new Object();
+
/**
* All keys, to allow reentrancy.
* This will grow forever with the number of applications seen, but this should be too slow to be a problem.
@@ -94,14 +96,42 @@ public class CuratorDatabase {
public Optional<byte[]> getData(Path path) { return getCache().getData(path); }
+ private static class CacheAndGeneration {
+ public CacheAndGeneration(CuratorDatabaseCache cache, long generation)
+ {
+ this.cache = cache;
+ this.generation = generation;
+ }
+ public boolean expired() {
+ return generation != cache.generation();
+ }
+ public CuratorDatabaseCache validCache() {
+ if (expired()) {
+ throw new IllegalStateException("The cache has generation " + cache.generation() +
+ " while the root genration counter in zookeeper says " + generation +
+ ". That is totally unacceptable and must be a sever programming error in my close vicinity.");
+ }
+ return cache;
+ }
+
+ private CuratorDatabaseCache cache;
+ private long generation;
+ }
+ private CacheAndGeneration getCacheSnapshot() {
+ return new CacheAndGeneration(cache.get(), changeGenerationCounter.get());
+ }
private CuratorDatabaseCache getCache() {
- CuratorDatabaseCache cache = this.cache.get();
- long currentCuratorGeneration = changeGenerationCounter.get();
- if (currentCuratorGeneration != cache.generation()) { // current cache is invalid - start new
- cache = newCache(currentCuratorGeneration);
- this.cache.set(cache);
+ CacheAndGeneration cacheAndGeneration = getCacheSnapshot();
+ while (cacheAndGeneration.expired()) {
+ synchronized (cacheCreationLock) { // Prevent a race for creating new caches
+ cacheAndGeneration = getCacheSnapshot();
+ if (cacheAndGeneration.expired()) {
+ cache.set(newCache(changeGenerationCounter.get()));
+ cacheAndGeneration = getCacheSnapshot();
+ }
+ }
}
- return cache;
+ return cacheAndGeneration.validCache();
}
/** Caches must only be instantiated using this method */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java
index 6d92f9c4541..bea7973541a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java
@@ -1,6 +1,8 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.restapi.v2.filter;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId;
@@ -11,6 +13,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
import java.security.cert.X509Certificate;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static com.yahoo.vespa.athenz.tls.SubjectAlternativeName.Type.DNS_NAME;
@@ -31,9 +34,13 @@ class NodeIdentifier {
private final Zone zone;
private final NodeRepository nodeRepository;
+
+ private final Supplier<List<Node>> nodeCache;
+
NodeIdentifier(Zone zone, NodeRepository nodeRepository) {
this.zone = zone;
this.nodeRepository = nodeRepository;
+ nodeCache = Suppliers.memoizeWithExpiration(nodeRepository::getNodes, 1, TimeUnit.MINUTES);
}
NodePrincipal resolveNode(List<X509Certificate> certificateChain) throws NodeIdentifierException {
@@ -73,7 +80,7 @@ class NodeIdentifier {
private String getHostFromCalypsoCertificate(List<SubjectAlternativeName> sans) {
String openstackId = getUniqueInstanceId(sans);
- return nodeRepository.getNodes().stream()
+ return nodeCache.get().stream()
.filter(node -> node.openStackId().equals(openstackId))
.map(Node::hostname)
.findFirst()
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java
index c0cead74f5f..11c7832091b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java
@@ -29,6 +29,7 @@ import java.security.cert.X509Certificate;
import java.time.Instant;
import java.util.Optional;
+import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.*;
import static com.yahoo.vespa.athenz.tls.KeyAlgorithm.RSA;
import static com.yahoo.vespa.athenz.tls.SignatureAlgorithm.SHA256_WITH_RSA;
import static java.util.Collections.emptySet;
@@ -161,7 +162,7 @@ public class NodeIdentifierTest {
Pkcs10Csr csr = Pkcs10CsrBuilder
.fromKeypair(new X500Principal("CN=" + TENANT_NODE_IDENTITY), KEYPAIR, SHA256_WITH_RSA)
.build();
- VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(clusterIndex, clusterId, INSTANCE_ID, application, tenant, region, environment);
+ VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(clusterIndex, clusterId, INSTANCE_ID, application, tenant, region, environment, NODE);
X509Certificate certificate = X509CertificateBuilder
.fromCsr(csr, ATHENZ_YAHOO_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_RSA, 1)
.addSubjectAlternativeName(vespaUniqueInstanceId.asDottedString() + ".instanceid.athenz.provider-name.vespa.yahoo.cloud")
diff --git a/pom.xml b/pom.xml
index 27063a6ffcf..1f766b2878d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,6 +72,7 @@
<module>documentapi</module>
<module>document</module>
<module>documentgen-test</module>
+ <module>fat-model-dependencies</module>
<module>fileacquirer</module>
<module>filedistribution</module>
<module>fsa</module>
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index ee5f29255fb..634f69a3820 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -1,6 +1,4 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("attribute_test");
#include <vespa/config-attributes.h>
#include <vespa/document/fieldvalue/document.h>
@@ -15,6 +13,7 @@ LOG_SETUP("attribute_test");
#include <vespa/searchcommon/attribute/attributecontent.h>
#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h>
#include <vespa/searchcore/proton/attribute/attributemanager.h>
#include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
#include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
@@ -44,6 +43,9 @@ LOG_SETUP("attribute_test");
#include <vespa/vespalib/testkit/testapp.h>
#include <vespa/searchcommon/attribute/iattributevector.h>
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_test");
+
namespace vespa { namespace config { namespace search {}}}
using namespace config;
@@ -139,6 +141,7 @@ struct Fixture
: Fixture(1)
{
}
+ ~Fixture();
void allocAttributeWriter() {
_aw = std::make_unique<AttributeWriter>(_m);
}
@@ -155,8 +158,12 @@ struct Fixture
_aw->put(serialNum, doc, lid, immediateCommit, emptyCallback);
}
void update(SerialNum serialNum, const DocumentUpdate &upd,
+ DocumentIdT lid, bool immediateCommit, IFieldUpdateCallback & onUpdate) {
+ _aw->update(serialNum, upd, lid, immediateCommit, emptyCallback, onUpdate);
+ }
+ void update(SerialNum serialNum, const Document &doc,
DocumentIdT lid, bool immediateCommit) {
- _aw->update(serialNum, upd, lid, immediateCommit, emptyCallback);
+ _aw->update(serialNum, doc, lid, immediateCommit, emptyCallback);
}
void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit = true) {
_aw->remove(serialNum, lid, immediateCommit, emptyCallback);
@@ -172,6 +179,7 @@ struct Fixture
}
};
+Fixture::~Fixture() = default;
TEST_F("require that attribute writer handles put", Fixture)
{
@@ -442,8 +450,9 @@ TEST_F("require that attribute writer handles update", Fixture)
upd.addUpdate(FieldUpdate(upd.getType().getField("a2"))
.addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)));
+ DummyFieldUpdateCallback onUpdate;
bool immediateCommit = true;
- f.update(2, upd, 1, immediateCommit);
+ f.update(2, upd, 1, immediateCommit, onUpdate);
attribute::IntegerContent ibuf;
ibuf.fill(*a1, 1);
@@ -453,9 +462,9 @@ TEST_F("require that attribute writer handles update", Fixture)
EXPECT_EQUAL(1u, ibuf.size());
EXPECT_EQUAL(30u, ibuf[0]);
- f.update(2, upd, 1, immediateCommit); // same sync token as previous
+ f.update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous
try {
- f.update(1, upd, 1, immediateCommit); // lower sync token than previous
+ f.update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous
EXPECT_TRUE(true); // update is ignored
} catch (vespalib::IllegalStateException & e) {
LOG(info, "Got expected exception: '%s'", e.getMessage().c_str());
@@ -488,7 +497,8 @@ TEST_F("require that attribute writer handles predicate update", Fixture)
EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size());
EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid());
bool immediateCommit = true;
- f.update(2, upd, 1, immediateCommit);
+ DummyFieldUpdateCallback onUpdate;
+ f.update(2, upd, 1, immediateCommit, onUpdate);
EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size());
EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid());
}
@@ -675,7 +685,8 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture)
upd.addUpdate(FieldUpdate(upd.getType().getField("a1"))
.addUpdate(AssignValueUpdate(new_value)));
bool immediateCommit = true;
- f.update(2, upd, 1, immediateCommit);
+ DummyFieldUpdateCallback onUpdate;
+ f.update(2, upd, 1, immediateCommit, onUpdate);
EXPECT_EQUAL(2u, a1->getNumDocs());
EXPECT_TRUE(tensorAttribute != nullptr);
tensor2 = tensorAttribute->getTensor(1);
@@ -773,6 +784,158 @@ TEST_F("require that AttributeWriter::forceCommit() clears search cache in impor
EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size());
}
+struct StructFixtureBase : public Fixture
+{
+ DocumentType _type;
+ const Field _valueField;
+ StructDataType _structFieldType;
+
+ StructFixtureBase()
+ : Fixture(),
+ _type("test"),
+ _valueField("value", 2, *DataType::INT, true),
+ _structFieldType("struct")
+ {
+ addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}, createSerialNum);
+ _type.addField(_valueField);
+ _structFieldType.addField(_valueField);
+ }
+ ~StructFixtureBase();
+
+ std::unique_ptr<StructFieldValue>
+ makeStruct()
+ {
+ return std::make_unique<StructFieldValue>(_structFieldType);
+ }
+
+ std::unique_ptr<StructFieldValue>
+ makeStruct(const int32_t value)
+ {
+ auto ret = makeStruct();
+ ret->setValue(_valueField, IntFieldValue(value));
+ return ret;
+ }
+
+ std::unique_ptr<Document>
+ makeDoc()
+ {
+ return std::make_unique<Document>(_type, DocumentId("id::test::1"));
+ }
+};
+
+StructFixtureBase::~StructFixtureBase() = default;
+
+struct StructArrayFixture : public StructFixtureBase
+{
+ using StructFixtureBase::makeDoc;
+ const ArrayDataType _structArrayFieldType;
+ const Field _structArrayField;
+
+ StructArrayFixture()
+ : StructFixtureBase(),
+ _structArrayFieldType(_structFieldType),
+ _structArrayField("array", _structArrayFieldType, true)
+ {
+ addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum);
+ _type.addField(_structArrayField);
+ }
+ ~StructArrayFixture();
+
+ std::unique_ptr<Document>
+ makeDoc(int32_t value, const std::vector<int32_t> &arrayValues)
+ {
+ auto doc = makeDoc();
+ doc->setValue(_valueField, IntFieldValue(value));
+ ArrayFieldValue s(_structArrayFieldType);
+ for (const auto &arrayValue : arrayValues) {
+ s.add(*makeStruct(arrayValue));
+ }
+ doc->setValue(_structArrayField, s);
+ return doc;
+ }
+ void checkAttrs(uint32_t lid, int32_t value, const std::vector<int32_t> &arrayValues) {
+ auto valueAttr = _m->getAttribute("value")->getSP();
+ auto arrayValueAttr = _m->getAttribute("array.value")->getSP();
+ EXPECT_EQUAL(value, valueAttr->getInt(lid));
+ attribute::IntegerContent ibuf;
+ ibuf.fill(*arrayValueAttr, lid);
+ EXPECT_EQUAL(arrayValues.size(), ibuf.size());
+ for (size_t i = 0; i < arrayValues.size(); ++i) {
+ EXPECT_EQUAL(arrayValues[i], ibuf[i]);
+ }
+ }
+};
+
+StructArrayFixture::~StructArrayFixture() = default;
+
+TEST_F("require that update with doc argument updates compound attributes (array)", StructArrayFixture)
+{
+ auto doc = f.makeDoc(10, {11, 12});
+ f.put(10, *doc, 1);
+ TEST_DO(f.checkAttrs(1, 10, {11, 12}));
+ doc = f.makeDoc(20, {21});
+ f.update(11, *doc, 1, true);
+ TEST_DO(f.checkAttrs(1, 10, {21}));
+}
+
+struct StructMapFixture : public StructFixtureBase
+{
+ using StructFixtureBase::makeDoc;
+ const MapDataType _structMapFieldType;
+ const Field _structMapField;
+
+ StructMapFixture()
+ : StructFixtureBase(),
+ _structMapFieldType(*DataType::INT, _structFieldType),
+ _structMapField("map", _structMapFieldType, true)
+ {
+ addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum);
+ addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum);
+ _type.addField(_structMapField);
+ }
+
+ std::unique_ptr<Document>
+ makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues)
+ {
+ auto doc = makeDoc();
+ doc->setValue(_valueField, IntFieldValue(value));
+ MapFieldValue s(_structMapFieldType);
+ for (const auto &mapValue : mapValues) {
+ s.put(IntFieldValue(mapValue.first), *makeStruct(mapValue.second));
+ }
+ doc->setValue(_structMapField, s);
+ return doc;
+ }
+ void checkAttrs(uint32_t lid, int32_t expValue, const std::map<int32_t, int32_t> &expMap) {
+ auto valueAttr = _m->getAttribute("value")->getSP();
+ auto mapKeyAttr = _m->getAttribute("map.key")->getSP();
+ auto mapValueAttr = _m->getAttribute("map.value.value")->getSP();
+ EXPECT_EQUAL(expValue, valueAttr->getInt(lid));
+ attribute::IntegerContent mapKeys;
+ mapKeys.fill(*mapKeyAttr, lid);
+ attribute::IntegerContent mapValues;
+ mapValues.fill(*mapValueAttr, lid);
+ EXPECT_EQUAL(expMap.size(), mapValues.size());
+ EXPECT_EQUAL(expMap.size(), mapKeys.size());
+ size_t i = 0;
+ for (const auto &expMapElem : expMap) {
+ EXPECT_EQUAL(expMapElem.first, mapKeys[i]);
+ EXPECT_EQUAL(expMapElem.second, mapValues[i]);
+ ++i;
+ }
+ }
+};
+
+TEST_F("require that update with doc argument updates compound attributes (map)", StructMapFixture)
+{
+ auto doc = f.makeDoc(10, {{1, 11}, {2, 12}});
+ f.put(10, *doc, 1);
+ TEST_DO(f.checkAttrs(1, 10, {{1, 11}, {2, 12}}));
+ doc = f.makeDoc(20, {{42, 21}});
+ f.update(11, *doc, 1, true);
+ TEST_DO(f.checkAttrs(1, 10, {{42, 21}}));
+}
+
TEST_MAIN()
{
vespalib::rmdir(test_dir, true);
diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp
index d13e4207a9d..bad27938d4b 100644
--- a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp
+++ b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp
@@ -86,19 +86,6 @@ makeStringWeightedSet(const std::vector<std::pair<vespalib::string, int32_t>> &a
return result;
}
-template <typename F1, typename F2>
-void
-checkFieldPathChange(F1 f1, F2 f2, const vespalib::string &path, bool same)
-{
- FieldPath fieldPath1 = f1.makeFieldPath(path);
- FieldPath fieldPath2 = f2.makeFieldPath(path);
- EXPECT_TRUE(!fieldPath1.empty());
- EXPECT_TRUE(!fieldPath2.empty());
- EXPECT_TRUE(DocumentFieldExtractor::isSupported(fieldPath1));
- EXPECT_TRUE(DocumentFieldExtractor::isSupported(fieldPath2));
- EXPECT_EQUAL(same, DocumentFieldExtractor::isCompatible(fieldPath1, fieldPath2));
-}
-
}
struct FixtureBase
@@ -354,26 +341,6 @@ TEST_F("require that unknown field gives null value", FixtureBase(false))
TEST_DO(f.assertExtracted("unknown", std::unique_ptr<FieldValue>()));
}
-TEST("require that type changes are detected")
-{
- TEST_DO(checkFieldPathChange(SimpleFixture(false), SimpleFixture(false), "weight", true));
- TEST_DO(checkFieldPathChange(SimpleFixture(false), SimpleFixture(true), "weight", false));
- TEST_DO(checkFieldPathChange(ArrayFixture(false), ArrayFixture(false), "weight", true));
- TEST_DO(checkFieldPathChange(ArrayFixture(false), ArrayFixture(true), "weight", false));
- TEST_DO(checkFieldPathChange(SimpleFixture(false), ArrayFixture(false), "weight", false));
- TEST_DO(checkFieldPathChange(WeightedSetFixture(false), WeightedSetFixture(false), "weight", true));
- TEST_DO(checkFieldPathChange(WeightedSetFixture(false), WeightedSetFixture(true), "weight", false));
- TEST_DO(checkFieldPathChange(SimpleFixture(false), WeightedSetFixture(false), "weight", false));
- TEST_DO(checkFieldPathChange(ArrayFixture(false), WeightedSetFixture(false), "weight", false));
- TEST_DO(checkFieldPathChange(StructArrayFixture(false), StructArrayFixture(false), "s.weight", true));
- TEST_DO(checkFieldPathChange(StructArrayFixture(false), StructArrayFixture(true), "s.weight", false));
- TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, false), "s.value.weight", true));
- TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(true, false), "s.value.weight", false));
- TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, true), "s.value.weight", false));
- TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, false), "s.key", true));
- TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, true), "s.key", false));
-}
-
TEST_MAIN()
{
TEST_RUN_ALL();
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index 00eb59f120a..5d040024e63 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include <vespa/searchcore/proton/attribute/i_attribute_writer.h>
+#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h>
#include <vespa/searchcore/proton/test/bucketfactory.h>
#include <vespa/searchcore/proton/common/commit_time_tracker.h>
#include <vespa/searchcore/proton/common/feedtoken.h>
@@ -316,21 +317,23 @@ struct MyAttributeWriter : public IAttributeWriter
std::set<vespalib::string> _attrs;
proton::IAttributeManager::SP _mgr;
MyTracer &_tracer;
+
MyAttributeWriter(MyTracer &tracer);
~MyAttributeWriter();
- virtual std::vector<AttributeVector *>
+
+ std::vector<AttributeVector *>
getWritableAttributes() const override {
return std::vector<AttributeVector *>();
}
- virtual AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override {
+ AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override {
if (_attrs.count(attrName) == 0) {
return nullptr;
}
AttrMap::const_iterator itr = _attrMap.find(attrName);
return ((itr == _attrMap.end()) ? nullptr : itr->second.get());
}
- virtual void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid,
- bool immediateCommit, OnWriteDoneType) override {
+ void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid,
+ bool immediateCommit, OnWriteDoneType) override {
_putSerial = serialNum;
_putDocId = doc.getId();
_putLid = lid;
@@ -339,8 +342,8 @@ struct MyAttributeWriter : public IAttributeWriter
++_commitCount;
}
}
- virtual void remove(SerialNum serialNum, DocumentIdT lid,
- bool immediateCommit, OnWriteDoneType) override {
+ void remove(SerialNum serialNum, DocumentIdT lid,
+ bool immediateCommit, OnWriteDoneType) override {
_removeSerial = serialNum;
_removeLid = lid;
_tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit);
@@ -348,37 +351,45 @@ struct MyAttributeWriter : public IAttributeWriter
++_commitCount;
}
}
- virtual void remove(const LidVector & lidsToRemove, SerialNum serialNum,
- bool immediateCommit, OnWriteDoneType) override {
+ void remove(const LidVector & lidsToRemove, SerialNum serialNum,
+ bool immediateCommit, OnWriteDoneType) override {
for (uint32_t lid : lidsToRemove) {
LOG(info, "MyAttributeAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)", serialNum, lid);
_removes.push_back(lid);
_tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit);
}
}
- virtual void update(SerialNum serialNum, const document::DocumentUpdate &upd,
- DocumentIdT lid, bool, OnWriteDoneType) override {
+ void update(SerialNum serialNum, const document::DocumentUpdate &upd,
+ DocumentIdT lid, bool, OnWriteDoneType, IFieldUpdateCallback & onUpdate) override {
_updateSerial = serialNum;
_updateDocId = upd.getId();
_updateLid = lid;
+ for (const auto & fieldUpdate : upd.getUpdates()) {
+ search::AttributeVector * attr = getWritableAttribute(fieldUpdate.getField().getName());
+ onUpdate.onUpdateField(fieldUpdate.getField().getName(), attr);
+ }
}
- virtual void heartBeat(SerialNum) override { ++_heartBeatCount; }
- virtual void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override {
+ void update(SerialNum serialNum, const document::Document &doc, DocumentIdT lid,
+ bool immediateCommit, OnWriteDoneType) override {
(void) serialNum;
+ (void) doc;
+ (void) lid;
+ (void) immediateCommit;
+ }
+ void heartBeat(SerialNum) override { ++_heartBeatCount; }
+ void compactLidSpace(uint32_t wantedLidLimit, SerialNum ) override {
_wantedLidLimit = wantedLidLimit;
}
- virtual const proton::IAttributeManager::SP &getAttributeManager() const override {
+ const proton::IAttributeManager::SP &getAttributeManager() const override {
return _mgr;
}
void forceCommit(SerialNum serialNum, OnWriteDoneType) override {
- (void) serialNum; ++_commitCount;
+ ++_commitCount;
_tracer.traceCommit(attributeAdapterTypeName, serialNum);
}
- virtual void onReplayDone(uint32_t docIdLimit) override
- {
- (void) docIdLimit;
- }
+ void onReplayDone(uint32_t ) override { }
+ bool getHasCompoundAttribute() const override { return false; }
};
MyAttributeWriter::MyAttributeWriter(MyTracer &tracer)
@@ -396,7 +407,7 @@ MyAttributeWriter::MyAttributeWriter(MyTracer &tracer)
cfg3.setTensorType(ValueType::from_spec("tensor(x[10])"));
_attrMap["a3"] = search::AttributeFactory::createAttribute("test3", cfg3);
}
-MyAttributeWriter::~MyAttributeWriter() {}
+MyAttributeWriter::~MyAttributeWriter() = default;
struct MyTransport : public feedtoken::ITransport
{
@@ -420,7 +431,7 @@ struct MyResultHandler : public IGenericResultHandler
{
vespalib::Gate _gate;
MyResultHandler() : _gate() {}
- virtual void handle(const storage::spi::Result &) override {
+ void handle(const storage::spi::Result &) override {
_gate.countDown();
}
void await() { _gate.await(); }
@@ -446,7 +457,7 @@ SchemaContext::SchemaContext() :
_schema->addSummaryField(Schema::SummaryField("s1", DataType::STRING, CollectionType::SINGLE));
_builder.reset(new DocBuilder(*_schema));
}
-SchemaContext::~SchemaContext() {}
+SchemaContext::~SchemaContext() = default;
struct DocumentContext
diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
index a6d7f12f199..681d3543ee1 100644
--- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
+++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
@@ -23,6 +23,7 @@
#include <vespa/document/update/documentupdate.h>
#include <vespa/document/update/assignvalueupdate.h>
#include <vespa/document/fieldvalue/fieldvalues.h>
+#include <vespa/document/serialization/vespadocumentserializer.h>
#include <vespa/document/repo/configbuilder.h>
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/document/datatype/documenttype.h>
@@ -282,8 +283,10 @@ TEST_F("require that we can deserialize old update operations", Fixture)
BucketId bucket(toBucket(docId.getGlobalId()));
auto upd(f.makeUpdate());
{
- UpdateOperation op(UpdateOperation::makeOldUpdate(bucket, Timestamp(10), upd));
- op.serialize(stream);
+ UpdateOperation op(bucket, Timestamp(10), upd);
+ op.serializeDocumentOperationOnly(stream);
+ document::VespaDocumentSerializer serializer(stream);
+ serializer.write42(*op.getUpdate());
}
{
UpdateOperation op(FeedOperation::UPDATE_42);
diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp
index 62c61406f67..a770fff3f5f 100644
--- a/searchcore/src/tests/proton/matching/matching_test.cpp
+++ b/searchcore/src/tests/proton/matching/matching_test.cpp
@@ -61,6 +61,36 @@ void inject_match_phase_limiting(Properties &setup, const vespalib::string &attr
setup.import(cfg);
}
+FakeResult make_elem_result(const std::vector<std::pair<uint32_t,std::vector<uint32_t> > > &match_data) {
+ FakeResult result;
+ uint32_t pos_should_be_ignored = 0;
+ for (const auto &doc: match_data) {
+ result.doc(doc.first);
+ for (const auto &elem: doc.second) {
+ result.elem(elem).pos(++pos_should_be_ignored);
+ }
+ }
+ return result;
+}
+
+vespalib::string make_simple_stack_dump(const vespalib::string &field,
+ const vespalib::string &term)
+{
+ QueryBuilder<ProtonNodeTypes> builder;
+ builder.addStringTerm(term, field, 1, search::query::Weight(1));
+ return StackDumpCreator::create(*builder.build());
+}
+
+vespalib::string make_same_element_stack_dump(const vespalib::string &a1_term,
+ const vespalib::string &f1_term)
+{
+ QueryBuilder<ProtonNodeTypes> builder;
+ builder.addSameElement(2, "ignored field name");
+ builder.addStringTerm(a1_term, "a1", 1, search::query::Weight(1));
+ builder.addStringTerm(f1_term, "f1", 2, search::query::Weight(1));
+ return StackDumpCreator::create(*builder.build());
+}
+
//-----------------------------------------------------------------------------
const uint32_t NUM_DOCS = 1000;
@@ -238,6 +268,13 @@ struct MyWorld {
searchContext.attr().addResult("a1", term, result);
}
+ void add_same_element_results(const vespalib::string &a1_term, const vespalib::string &f1_0_term) {
+ auto a1_result = make_elem_result({{10, {1}}, {20, {2}}, {21, {2}}});
+ auto f1_0_result = make_elem_result({{10, {2}}, {20, {2}}, {21, {2}}});
+ searchContext.attr().addResult("a1", a1_term, a1_result);
+ searchContext.idx(0).getFake().addResult("f1", f1_0_term, f1_0_result);
+ }
+
void basicResults() {
searchContext.idx(0).getFake().addResult("f1", "foo",
FakeResult()
@@ -249,26 +286,32 @@ struct MyWorld {
.doc(600).doc(700).doc(800).doc(900));
}
- void setStackDump(Request &request, const vespalib::string &field,
- const vespalib::string &term) {
- QueryBuilder<ProtonNodeTypes> builder;
- builder.addStringTerm(term, field, 1, search::query::Weight(1));
- vespalib::string stack_dump =
- StackDumpCreator::create(*builder.build());
+ void setStackDump(Request &request, const vespalib::string &stack_dump) {
request.stackDump.assign(stack_dump.data(),
stack_dump.data() + stack_dump.size());
}
- SearchRequest::SP createSimpleRequest(const vespalib::string &field,
- const vespalib::string &term)
+ SearchRequest::SP createRequest(const vespalib::string &stack_dump)
{
SearchRequest::SP request(new SearchRequest);
request->setTimeout(60 * fastos::TimeStamp::SEC);
- setStackDump(*request, field, term);
+ setStackDump(*request, stack_dump);
request->maxhits = 10;
return request;
}
+ SearchRequest::SP createSimpleRequest(const vespalib::string &field,
+ const vespalib::string &term)
+ {
+ return createRequest(make_simple_stack_dump(field, term));
+ }
+
+ SearchRequest::SP createSameElementRequest(const vespalib::string &a1_term,
+ const vespalib::string &f1_term)
+ {
+ return createRequest(make_same_element_stack_dump(a1_term, f1_term));
+ }
+
Matcher::SP createMatcher() {
return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, 0);
}
@@ -317,7 +360,7 @@ struct MyWorld {
const vespalib::string & term)
{
DocsumRequest::SP request(new DocsumRequest);
- setStackDump(*request, field, term);
+ setStackDump(*request, make_simple_stack_dump(field, term));
// match a subset of basic result + request for a non-hit (not
// sorted on docid)
@@ -800,4 +843,14 @@ TEST("require that fields are tagged with data type") {
EXPECT_EQUAL(predicate_field->get_data_type(), FieldInfo::DataType::BOOLEANTREE);
}
+TEST("require that same element search works (note that this does not test/use the attribute element iterator wrapper)") {
+ MyWorld world;
+ world.basicSetup();
+ world.add_same_element_results("foo", "bar");
+ SearchRequest::SP request = world.createSameElementRequest("foo", "bar");
+ SearchReply::UP reply = world.performSearch(request, 1);
+ ASSERT_EQUAL(1u, reply->hits.size());
+ EXPECT_EQUAL(document::DocumentId("doc::20").getGlobalId(), reply->hits[0].gid);
+}
+
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
index 9262f9a7b6f..705a27c7fc3 100644
--- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
+++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
@@ -68,24 +68,8 @@ createUpd(const DocumentType& docType, const DocumentId &docId)
return document::DocumentUpdate::SP(new document::DocumentUpdate(docType, docId));
}
-
-document::Document::UP
-clone(const document::Document::SP &doc)
-{
- return document::Document::UP(doc->clone());
-}
-
-
-document::DocumentUpdate::UP
-clone(const document::DocumentUpdate::SP &upd)
-{
- return document::DocumentUpdate::UP(upd->clone());
-}
-
-
storage::spi::ClusterState
-createClusterState(const storage::lib::State& nodeState =
- storage::lib::State::UP)
+createClusterState(const storage::lib::State& nodeState = storage::lib::State::UP)
{
using storage::lib::Distribution;
using storage::lib::Node;
@@ -99,11 +83,7 @@ createClusterState(const storage::lib::State& nodeState =
StorDistributionConfigBuilder dc;
cstate.setNodeState(Node(NodeType::STORAGE, 0),
- NodeState(NodeType::STORAGE,
- nodeState,
- "dummy desc",
- 1.0,
- 1));
+ NodeState(NodeType::STORAGE, nodeState, "dummy desc", 1.0, 1));
cstate.setClusterState(State::UP);
dc.redundancy = 1;
dc.readyCopies = 1;
@@ -222,8 +202,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer {
void handleUpdate(FeedToken token, const Bucket& bucket,
Timestamp timestamp, const document::DocumentUpdate::SP& upd) override {
- token->setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)),
- existingTimestamp > 0);
+ token->setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)), existingTimestamp > 0);
handle(token, bucket, timestamp, upd->getId());
}
@@ -312,8 +291,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer {
return frozen.find(bucket.getBucketId().getId()) != frozen.end();
}
bool wasFrozen(const Bucket &bucket) {
- return was_frozen.find(bucket.getBucketId().getId())
- != was_frozen.end();
+ return was_frozen.find(bucket.getBucketId().getId()) != was_frozen.end();
}
};
@@ -335,7 +313,7 @@ HandlerSet::HandlerSet()
handler1(static_cast<MyHandler &>(*phandler1.get())),
handler2(static_cast<MyHandler &>(*phandler2.get()))
{}
-HandlerSet::~HandlerSet() {}
+HandlerSet::~HandlerSet() = default;
DocumentType type1(createDocType("type1", 1));
DocumentType type2(createDocType("type2", 2));
@@ -405,8 +383,8 @@ struct SimpleResourceWriteFilter : public IResourceWriteFilter
_message()
{}
- virtual bool acceptWriteOperation() const override { return _acceptWriteOperation; }
- virtual State getAcceptState() const override {
+ bool acceptWriteOperation() const override { return _acceptWriteOperation; }
+ State getAcceptState() const override {
return IResourceWriteFilter::State(acceptWriteOperation(), _message);
}
};
@@ -475,8 +453,7 @@ TEST_F("require that getPartitionStates() prepares all handlers", SimpleFixture)
TEST_F("require that puts are routed to handler", SimpleFixture)
{
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
f.engine.put(bucket1, tstamp1, doc1, context);
assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
assertHandler(bucket0, tstamp0, docId0, f.hset.handler2);
@@ -485,20 +462,16 @@ TEST_F("require that puts are routed to handler", SimpleFixture)
assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
assertHandler(bucket1, tstamp1, docId2, f.hset.handler2);
- EXPECT_EQUAL(
- Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"),
- f.engine.put(bucket1, tstamp1, doc3, context));
+ EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"),
+ f.engine.put(bucket1, tstamp1, doc3, context));
}
TEST_F("require that puts with old id scheme are rejected", SimpleFixture) {
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
- EXPECT_EQUAL(
- Result(Result::PERMANENT_ERROR, "Old id scheme not supported in "
- "elastic mode (doc:old:id-scheme)"),
- f.engine.put(bucket1, tstamp1, old_doc, context));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
+ EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "Old id scheme not supported in elastic mode (doc:old:id-scheme)"),
+ f.engine.put(bucket1, tstamp1, old_doc, context));
}
@@ -508,8 +481,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur
f._writeFilter._message = "Disk is full";
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
EXPECT_EQUAL(
Result(Result::RESOURCE_EXHAUSTED,
"Put operation rejected for document 'doc:old:id-scheme': 'Disk is full'"),
@@ -520,8 +492,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur
TEST_F("require that updates are routed to handler", SimpleFixture)
{
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
f.hset.handler1.setExistingTimestamp(tstamp2);
UpdateResult ur = f.engine.update(bucket1, tstamp1, upd1, context);
assertHandler(bucket1, tstamp1, docId1, f.hset.handler1);
@@ -534,9 +505,8 @@ TEST_F("require that updates are routed to handler", SimpleFixture)
assertHandler(bucket1, tstamp1, docId2, f.hset.handler2);
EXPECT_EQUAL(tstamp3, ur.getExistingTimestamp());
- EXPECT_EQUAL(
- Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"),
- f.engine.update(bucket1, tstamp1, upd3, context));
+ EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"),
+ f.engine.update(bucket1, tstamp1, upd3, context));
}
@@ -546,8 +516,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix
f._writeFilter._message = "Disk is full";
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
EXPECT_EQUAL(
Result(Result::RESOURCE_EXHAUSTED,
@@ -559,8 +528,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix
TEST_F("require that removes are routed to handlers", SimpleFixture)
{
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
RemoveResult rr = f.engine.remove(bucket1, tstamp1, docId3, context);
assertHandler(bucket0, tstamp0, docId0, f.hset.handler1);
assertHandler(bucket0, tstamp0, docId0, f.hset.handler2);
@@ -598,8 +566,7 @@ TEST_F("require that remove is NOT rejected if resource limit is reached", Simpl
f._writeFilter._message = "Disk is full";
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
EXPECT_EQUAL(RemoveResult(false), f.engine.remove(bucket1, tstamp1, docId1, context));
}
@@ -628,8 +595,7 @@ TEST_F("require that setActiveState() is routed to handlers and merged", SimpleF
f.hset.handler1.bucketStateResult = Result(Result::TRANSIENT_ERROR, "err1");
f.hset.handler2.bucketStateResult = Result(Result::PERMANENT_ERROR, "err2");
- Result result = f.engine.setActiveState(bucket1,
- storage::spi::BucketInfo::NOT_ACTIVE);
+ Result result = f.engine.setActiveState(bucket1, storage::spi::BucketInfo::NOT_ACTIVE);
EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode());
EXPECT_EQUAL("err1, err2", result.getErrorMessage());
EXPECT_EQUAL(storage::spi::BucketInfo::NOT_ACTIVE, f.hset.handler1.lastBucketState);
@@ -651,16 +617,12 @@ TEST_F("require that getBucketInfo() is routed to handlers and merged", SimpleFi
}
-TEST_F("require that createBucket() is routed to handlers and merged",
- SimpleFixture)
+TEST_F("require that createBucket() is routed to handlers and merged", SimpleFixture)
{
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
- f.hset.handler1._createBucketResult =
- Result(Result::TRANSIENT_ERROR, "err1a");
- f.hset.handler2._createBucketResult =
- Result(Result::PERMANENT_ERROR, "err2a");
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
+ f.hset.handler1._createBucketResult = Result(Result::TRANSIENT_ERROR, "err1a");
+ f.hset.handler2._createBucketResult = Result(Result::PERMANENT_ERROR, "err2a");
Result result = f.engine.createBucket(bucket1, context);
EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode());
@@ -671,8 +633,7 @@ TEST_F("require that createBucket() is routed to handlers and merged",
TEST_F("require that deleteBucket() is routed to handlers and merged", SimpleFixture)
{
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
f.hset.handler1.deleteBucketResult = Result(Result::TRANSIENT_ERROR, "err1");
f.hset.handler2.deleteBucketResult = Result(Result::PERMANENT_ERROR, "err2");
@@ -691,10 +652,8 @@ TEST_F("require that getModifiedBuckets() is routed to handlers and merged", Sim
TEST_F("require that get is sent to all handlers", SimpleFixture) {
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
- GetResult result = f.engine.get(bucket1, document::AllFields(), docId1,
- context);
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
+ GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context);
EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId);
EXPECT_EQUAL(docId1, f.hset.handler2.lastDocId);
@@ -704,8 +663,7 @@ TEST_F("require that get freezes the bucket", SimpleFixture) {
EXPECT_FALSE(f.hset.handler1.wasFrozen(bucket1));
EXPECT_FALSE(f.hset.handler2.wasFrozen(bucket1));
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
f.engine.get(bucket1, document::AllFields(), docId1, context);
EXPECT_TRUE(f.hset.handler1.wasFrozen(bucket1));
EXPECT_TRUE(f.hset.handler2.wasFrozen(bucket1));
@@ -717,10 +675,8 @@ TEST_F("require that get returns the first document found", SimpleFixture) {
f.hset.handler1.setDocument(*doc1, tstamp1);
f.hset.handler2.setDocument(*doc2, tstamp2);
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
- GetResult result = f.engine.get(bucket1, document::AllFields(), docId1,
- context);
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
+ GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context);
EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId);
EXPECT_EQUAL(DocumentId(), f.hset.handler2.lastDocId);
@@ -732,8 +688,7 @@ TEST_F("require that get returns the first document found", SimpleFixture) {
TEST_F("require that createIterator does", SimpleFixture) {
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
CreateIteratorResult result =
f.engine.createIterator(bucket1, document::AllFields(), selection,
storage::spi::NEWEST_DOCUMENT_ONLY, context);
@@ -741,15 +696,13 @@ TEST_F("require that createIterator does", SimpleFixture) {
EXPECT_TRUE(result.getIteratorId());
uint64_t max_size = 1024;
- IterateResult it_result =
- f.engine.iterate(result.getIteratorId(), max_size, context);
+ IterateResult it_result = f.engine.iterate(result.getIteratorId(), max_size, context);
EXPECT_FALSE(it_result.hasError());
}
TEST_F("require that iterator ids are unique", SimpleFixture) {
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
CreateIteratorResult result =
f.engine.createIterator(bucket1, document::AllFields(), selection,
storage::spi::NEWEST_DOCUMENT_ONLY, context);
@@ -764,10 +717,8 @@ TEST_F("require that iterator ids are unique", SimpleFixture) {
TEST_F("require that iterate requires valid iterator", SimpleFixture) {
uint64_t max_size = 1024;
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
- IterateResult it_result = f.engine.iterate(IteratorId(1), max_size,
- context);
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
+ IterateResult it_result = f.engine.iterate(IteratorId(1), max_size, context);
EXPECT_TRUE(it_result.hasError());
EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode());
EXPECT_EQUAL("Unknown iterator with id 1", it_result.getErrorMessage());
@@ -786,16 +737,14 @@ TEST_F("require that iterate returns documents", SimpleFixture) {
f.hset.handler2.setDocument(*doc2, tstamp2);
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
uint64_t max_size = 1024;
CreateIteratorResult result =
f.engine.createIterator(bucket1, document::AllFields(), selection,
storage::spi::NEWEST_DOCUMENT_ONLY, context);
EXPECT_TRUE(result.getIteratorId());
- IterateResult it_result =
- f.engine.iterate(result.getIteratorId(), max_size, context);
+ IterateResult it_result = f.engine.iterate(result.getIteratorId(), max_size, context);
EXPECT_FALSE(it_result.hasError());
EXPECT_EQUAL(2u, it_result.getEntries().size());
}
@@ -804,33 +753,28 @@ TEST_F("require that destroyIterator prevents iteration", SimpleFixture) {
f.hset.handler1.setDocument(*doc1, tstamp1);
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
CreateIteratorResult create_result =
f.engine.createIterator(bucket1, document::AllFields(), selection,
storage::spi::NEWEST_DOCUMENT_ONLY, context);
EXPECT_TRUE(create_result.getIteratorId());
- Result result = f.engine.destroyIterator(create_result.getIteratorId(),
- context);
+ Result result = f.engine.destroyIterator(create_result.getIteratorId(), context);
EXPECT_FALSE(result.hasError());
uint64_t max_size = 1024;
- IterateResult it_result =
- f.engine.iterate(create_result.getIteratorId(), max_size, context);
+ IterateResult it_result = f.engine.iterate(create_result.getIteratorId(), max_size, context);
EXPECT_TRUE(it_result.hasError());
EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode());
string msg_prefix = "Unknown iterator with id";
- EXPECT_EQUAL(msg_prefix,
- it_result.getErrorMessage().substr(0, msg_prefix.size()));
+ EXPECT_EQUAL(msg_prefix, it_result.getErrorMessage().substr(0, msg_prefix.size()));
}
TEST_F("require that buckets are frozen during iterator life", SimpleFixture) {
EXPECT_FALSE(f.hset.handler1.isFrozen(bucket1));
EXPECT_FALSE(f.hset.handler2.isFrozen(bucket1));
storage::spi::LoadType loadType(0, "default");
- Context context(loadType, storage::spi::Priority(0),
- storage::spi::Trace::TraceLevel(0));
+ Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0));
CreateIteratorResult create_result =
f.engine.createIterator(bucket1, document::AllFields(), selection,
storage::spi::NEWEST_DOCUMENT_ONLY, context);
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index 0cb260ed9a8..35f5f09fc37 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "attribute_writer.h"
+#include "ifieldupdatecallback.h"
#include "attributemanager.h"
#include "document_field_extractor.h"
#include <vespa/document/base/exceptions.h>
@@ -22,10 +23,34 @@ namespace proton {
using LidVector = LidVectorContext::LidVector;
+AttributeWriter::WriteField::WriteField(AttributeVector &attribute)
+ : _fieldPath(),
+ _attribute(attribute),
+ _compoundAttribute(false)
+{
+ const vespalib::string &name = attribute.getName();
+ _compoundAttribute = name.find('.') != vespalib::string::npos;
+}
+
+AttributeWriter::WriteField::~WriteField() = default;
+
+void
+AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType)
+{
+ const vespalib::string &name = _attribute.getName();
+ FieldPath fp;
+ try {
+ docType.buildFieldPath(fp, name);
+ } catch (document::FieldNotFoundException & e) {
+ fp = FieldPath();
+ }
+ _fieldPath = std::move(fp);
+}
+
AttributeWriter::WriteContext::WriteContext(uint32_t executorId)
: _executorId(executorId),
- _fieldPaths(),
- _attributes()
+ _fields(),
+ _hasCompoundAttribute(false)
{
}
@@ -37,30 +62,20 @@ AttributeWriter::WriteContext::~WriteContext() = default;
AttributeWriter::WriteContext &AttributeWriter::WriteContext::operator=(WriteContext &&rhs) = default;
void
-AttributeWriter::WriteContext::add(AttributeVector *attr)
+AttributeWriter::WriteContext::add(AttributeVector &attr)
{
- _attributes.emplace_back(attr);
- _fieldPaths.emplace_back();
+ _fields.emplace_back(attr);
+ if (_fields.back().getCompoundAttribute()) {
+ _hasCompoundAttribute = true;
+ }
}
void
AttributeWriter::WriteContext::buildFieldPaths(const DocumentType &docType)
{
- size_t fieldId = 0;
- for (const auto &attrp : _attributes) {
- const vespalib::string &name = attrp->getName();
- FieldPath fp;
- try {
- docType.buildFieldPath(fp, name);
- } catch (document::FieldNotFoundException & e) {
- fp = FieldPath();
- }
-
- assert(fieldId < _fieldPaths.size());
- _fieldPaths[fieldId] = std::move(fp);
- ++fieldId;
+ for (auto &field : _fields) {
+ field.buildFieldPath(docType);
}
- assert(fieldId == _fieldPaths.size());
}
namespace {
@@ -200,26 +215,30 @@ class PutTask : public vespalib::Executor::Task
const SerialNum _serialNum;
const uint32_t _lid;
const bool _immediateCommit;
+ const bool _allAttributes;
std::remove_reference_t<AttributeWriter::OnWriteDoneType> _onWriteDone;
std::vector<FieldValue::UP> _fieldValues;
public:
- PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone);
+ PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone);
virtual ~PutTask() override;
virtual void run() override;
};
-PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone)
+PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone)
: _wc(wc),
_serialNum(serialNum),
_lid(lid),
_immediateCommit(immediateCommit),
+ _allAttributes(allAttributes),
_onWriteDone(onWriteDone)
{
- const auto &fieldPaths = _wc.getFieldPaths();
- _fieldValues.reserve(fieldPaths.size());
- for (const auto &fieldPath : fieldPaths) {
- FieldValue::UP fv = fieldExtractor.getFieldValue(fieldPath);
- _fieldValues.emplace_back(std::move(fv));
+ const auto &fields = _wc.getFields();
+ _fieldValues.reserve(fields.size());
+ for (const auto &field : fields) {
+ if (_allAttributes || field.getCompoundAttribute()) {
+ FieldValue::UP fv = fieldExtractor.getFieldValue(field.getFieldPath());
+ _fieldValues.emplace_back(std::move(fv));
+ }
}
}
@@ -231,13 +250,15 @@ void
PutTask::run()
{
uint32_t fieldId = 0;
- const auto &attributes = _wc.getAttributes();
- for (auto attrp : attributes) {
- AttributeVector &attr = *attrp;
- if (attr.getStatus().getLastSyncToken() < _serialNum) {
- applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, _immediateCommit, attr, _onWriteDone);
+ const auto &fields = _wc.getFields();
+ for (auto field : fields) {
+ if (_allAttributes || field.getCompoundAttribute()) {
+ AttributeVector &attr = field.getAttribute();
+ if (attr.getStatus().getLastSyncToken() < _serialNum) {
+ applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, _immediateCommit, attr, _onWriteDone);
+ }
+ ++fieldId;
}
- ++fieldId;
}
}
@@ -271,9 +292,9 @@ RemoveTask::~RemoveTask()
void
RemoveTask::run()
{
- const auto &attributes = _wc.getAttributes();
- for (auto &attrp : attributes) {
- AttributeVector &attr = *attrp;
+ const auto &fields = _wc.getFields();
+ for (auto &field : fields) {
+ AttributeVector &attr = field.getAttribute();
// Must use <= due to how move operations are handled
if (attr.getStatus().getLastSyncToken() <= _serialNum) {
applyRemoveToAttribute(_serialNum, _lid, _immediateCommit, attr, _onWriteDone);
@@ -303,13 +324,14 @@ public:
{}
virtual ~BatchRemoveTask() override {}
virtual void run() override {
- for (auto attr : _writeCtx.getAttributes()) {
- if (attr->getStatus().getLastSyncToken() < _serialNum) {
+ for (auto field : _writeCtx.getFields()) {
+ auto &attr = field.getAttribute();
+ if (attr.getStatus().getLastSyncToken() < _serialNum) {
for (auto lidToRemove : _lidsToRemove) {
- applyRemoveToAttribute(_serialNum, lidToRemove, false, *attr, _onWriteDone);
+ applyRemoveToAttribute(_serialNum, lidToRemove, false, attr, _onWriteDone);
}
if (_immediateCommit) {
- attr->commit(_serialNum, _serialNum);
+ attr.commit(_serialNum, _serialNum);
}
}
}
@@ -342,9 +364,9 @@ CommitTask::~CommitTask()
void
CommitTask::run()
{
- const auto &attributes = _wc.getAttributes();
- for (auto &attrp : attributes) {
- AttributeVector &attr = *attrp;
+ const auto &fields = _wc.getFields();
+ for (auto &field : fields) {
+ AttributeVector &attr = field.getAttribute();
applyCommit(_serialNum, _onWriteDone, attr);
}
}
@@ -365,7 +387,12 @@ AttributeWriter::setupWriteContexts()
(_writeContexts.back().getExecutorId() != fc.getExecutorId())) {
_writeContexts.emplace_back(fc.getExecutorId());
}
- _writeContexts.back().add(fc.getAttribute());
+ _writeContexts.back().add(*fc.getAttribute());
+ }
+ for (const auto &wc : _writeContexts) {
+ if (wc.getHasCompoundAttribute()) {
+ _hasCompoundAttribute = true;
+ }
}
}
@@ -380,12 +407,18 @@ AttributeWriter::buildFieldPaths(const DocumentType & docType, const DataType *d
void
AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid,
- bool immediateCommit, OnWriteDoneType onWriteDone)
+ bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone)
{
+ const DataType *dataType(doc.getDataType());
+ if (_dataType != dataType) {
+ buildFieldPaths(doc.getType(), dataType);
+ }
DocumentFieldExtractor extractor(doc);
for (const auto &wc : _writeContexts) {
- auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, onWriteDone);
- _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask));
+ if (allAttributes || wc.getHasCompoundAttribute()) {
+ auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, allAttributes, onWriteDone);
+ _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask));
+ }
}
}
@@ -405,7 +438,8 @@ AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr)
_attributeFieldWriter(mgr->getAttributeFieldWriter()),
_writableAttributes(mgr->getWritableAttributes()),
_writeContexts(),
- _dataType(nullptr)
+ _dataType(nullptr),
+ _hasCompoundAttribute(false)
{
setupWriteContexts();
}
@@ -432,18 +466,26 @@ void
AttributeWriter::put(SerialNum serialNum, const Document &doc, DocumentIdT lid,
bool immediateCommit, OnWriteDoneType onWriteDone)
{
- FieldValue::UP attrVal;
LOG(spam,
"Handle put: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)",
serialNum,
doc.getId().toString().c_str(),
lid,
doc.toString(true).c_str());
- const DataType *dataType(doc.getDataType());
- if (_dataType != dataType) {
- buildFieldPaths(doc.getType(), dataType);
- }
- internalPut(serialNum, doc, lid, immediateCommit, onWriteDone);
+ internalPut(serialNum, doc, lid, immediateCommit, true, onWriteDone);
+}
+
+void
+AttributeWriter::update(SerialNum serialNum, const Document &doc, DocumentIdT lid,
+ bool immediateCommit, OnWriteDoneType onWriteDone)
+{
+ LOG(spam,
+ "Handle update: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)",
+ serialNum,
+ doc.getId().toString().c_str(),
+ lid,
+ doc.toString(true).c_str());
+ internalPut(serialNum, doc, lid, immediateCommit, false, onWriteDone);
}
void
@@ -465,17 +507,15 @@ AttributeWriter::remove(const LidVector &lidsToRemove, SerialNum serialNum,
void
AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid,
- bool immediateCommit, OnWriteDoneType onWriteDone)
+ bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate)
{
LOG(debug, "Inspecting update for document %d.", lid);
for (const auto &fupd : upd.getUpdates()) {
- LOG(debug, "Retrieving guard for attribute vector '%s'.",
- fupd.getField().getName().c_str());
- AttributeVector *attrp =
- _mgr->getWritableAttribute(fupd.getField().getName());
+ LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().c_str());
+ AttributeVector *attrp = _mgr->getWritableAttribute(fupd.getField().getName());
+ onUpdate.onUpdateField(fupd.getField().getName(), attrp);
if (attrp == nullptr) {
- LOG(spam, "Failed to find attribute vector %s",
- fupd.getField().getName().c_str());
+ LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().c_str());
continue;
}
AttributeVector &attr = *attrp;
@@ -484,8 +524,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document
if (attr.getStatus().getLastSyncToken() >= serialNum)
continue;
- LOG(debug, "About to apply update for docId %u in attribute vector '%s'.",
- lid, attr.getName().c_str());
+ LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attr.getName().c_str());
// NOTE: The lifetime of the field update will be ensured by keeping the document update alive
// in a operation done context object.
@@ -550,5 +589,11 @@ AttributeWriter::compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum)
_attributeFieldWriter.sync();
}
+bool
+AttributeWriter::getHasCompoundAttribute() const
+{
+ return _hasCompoundAttribute;
+}
+
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
index cbda11180c0..dfdafa3bea9 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
@@ -23,30 +23,44 @@ private:
search::ISequencedTaskExecutor &_attributeFieldWriter;
const std::vector<search::AttributeVector *> &_writableAttributes;
public:
+ class WriteField
+ {
+ FieldPath _fieldPath;
+ AttributeVector &_attribute;
+ bool _compoundAttribute; // in array/map of struct
+ public:
+ WriteField(AttributeVector &attribute);
+ ~WriteField();
+ AttributeVector &getAttribute() const { return _attribute; }
+ const FieldPath &getFieldPath() const { return _fieldPath; }
+ void buildFieldPath(const DocumentType &docType);
+ bool getCompoundAttribute() const { return _compoundAttribute; }
+ };
class WriteContext
{
uint32_t _executorId;
- std::vector<FieldPath> _fieldPaths;
- std::vector<AttributeVector *> _attributes;
+ std::vector<WriteField> _fields;
+ bool _hasCompoundAttribute;
public:
WriteContext(uint32_t executorId);
WriteContext(WriteContext &&rhs);
~WriteContext();
WriteContext &operator=(WriteContext &&rhs);
void buildFieldPaths(const DocumentType &docType);
- void add(AttributeVector *attr);
+ void add(AttributeVector &attr);
uint32_t getExecutorId() const { return _executorId; }
- const std::vector<FieldPath> &getFieldPaths() const { return _fieldPaths; }
- const std::vector<AttributeVector *> &getAttributes() const { return _attributes; }
+ const std::vector<WriteField> &getFields() const { return _fields; }
+ bool getHasCompoundAttribute() const { return _hasCompoundAttribute; }
};
private:
std::vector<WriteContext> _writeContexts;
const DataType *_dataType;
+ bool _hasCompoundAttribute;
void setupWriteContexts();
void buildFieldPaths(const DocumentType &docType, const DataType *dataType);
void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid,
- bool immediateCommit, OnWriteDoneType onWriteDone);
+ bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone);
void internalRemove(SerialNum serialNum, DocumentIdT lid,
bool immediateCommit, OnWriteDoneType onWriteDone);
@@ -68,6 +82,8 @@ public:
void remove(const LidVector &lidVector, SerialNum serialNum,
bool immediateCommit, OnWriteDoneType onWriteDone) override;
void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid,
+ bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) override;
+ void update(SerialNum serialNum, const Document &doc, DocumentIdT lid,
bool immediateCommit, OnWriteDoneType onWriteDone) override;
void heartBeat(SerialNum serialNum) override;
void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override;
@@ -76,7 +92,8 @@ public:
}
void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) override;
- virtual void onReplayDone(uint32_t docIdLimit) override;
+ void onReplayDone(uint32_t docIdLimit) override;
+ bool getHasCompoundAttribute() const override;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp
index 143441eaae9..46f3fdeff67 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp
@@ -146,23 +146,6 @@ DocumentFieldExtractor::isSupported(const FieldPath &fieldPath)
return true;
}
-bool
-DocumentFieldExtractor::isCompatible(const FieldPath &fieldPath1, const FieldPath &fieldPath2)
-{
- if (fieldPath1.size() != fieldPath2.size()) {
- return false;
- }
- uint32_t arrayIndex = 0;
- for (const auto &fieldPathEntry1 : fieldPath1) {
- const auto &fieldPathEntry2 = fieldPath2[arrayIndex++];
- if (fieldPathEntry1->getType() != fieldPathEntry2.getType() ||
- fieldPathEntry1->getDataType() != fieldPathEntry2.getDataType()) {
- return false;
- }
- }
- return true;
-}
-
const FieldValue *
DocumentFieldExtractor::getCachedFieldValue(const FieldPathEntry &fieldPathEntry)
{
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h
index 71f020a582c..48e2da9c4c6 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h
@@ -41,15 +41,6 @@ public:
* Check if fieldPath is in a supported form.
*/
static bool isSupported(const document::FieldPath &fieldPath);
-
- /**
- * Check if two field paths are compatible, i.e. same types in whole path
- * and same data type would be returned from getFieldValue(). This is
- * meant to be used when document type in received document doesn't match
- * the document type for the current config (can happen right before and
- * after live config change when validation override is used).
- */
- static bool isCompatible(const document::FieldPath &fieldPath1, const document::FieldPath &fieldPath2);
};
}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
index abcf132d537..7a557b17964 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
@@ -1,18 +1,20 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
-#include <vespa/document/fieldvalue/document.h>
-#include <vespa/document/update/documentupdate.h>
+#include "i_attribute_manager.h"
+#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
#include <vespa/searchlib/attribute/attributeguard.h>
#include <vespa/searchlib/query/base.h>
#include <vespa/searchlib/common/serialnum.h>
-#include <vespa/searchcore/proton/attribute/i_attribute_manager.h>
-#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
namespace search { class IDestructorCallback; }
namespace proton {
+class IFieldUpdateCallback;
+
/**
* Interface for an attribute writer that handles writes in form of put, update and remove
* to an underlying set of attribute vectors.
@@ -31,10 +33,8 @@ public:
virtual ~IAttributeWriter() {}
- virtual std::vector<search::AttributeVector *>
- getWritableAttributes() const = 0;
- virtual search::AttributeVector *
- getWritableAttribute(const vespalib::string &attrName) const = 0;
+ virtual std::vector<search::AttributeVector *> getWritableAttributes() const = 0;
+ virtual search::AttributeVector *getWritableAttribute(const vespalib::string &attrName) const = 0;
virtual void put(SerialNum serialNum, const Document &doc, DocumentIdT lid,
bool immediateCommit, OnWriteDoneType onWriteDone) = 0;
virtual void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit,
@@ -46,6 +46,11 @@ public:
* The OnWriteDoneType instance should ensure the lifetime of the given DocumentUpdate instance.
*/
virtual void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid,
+ bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) = 0;
+ /*
+ * Update the underlying compound attributes based on updated document.
+ */
+ virtual void update(SerialNum serialNum, const Document &doc, DocumentIdT lid,
bool immediateCommit, OnWriteDoneType onWriteDone) = 0;
virtual void heartBeat(SerialNum serialNum) = 0;
/**
@@ -60,6 +65,8 @@ public:
virtual void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0;
virtual void onReplayDone(uint32_t docIdLimit) = 0;
+
+ virtual bool getHasCompoundAttribute() const = 0;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h b/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h
new file mode 100644
index 00000000000..ffb8555cd2c
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h
@@ -0,0 +1,20 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+
+namespace search { class AttributeVector; }
+
+namespace proton {
+
+struct IFieldUpdateCallback {
+ virtual ~IFieldUpdateCallback() { }
+ virtual void onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) = 0;
+};
+
+struct DummyFieldUpdateCallback : IFieldUpdateCallback {
+ void onUpdateField(vespalib::stringref, const search::AttributeVector *) override {}
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
index b2567527560..37b23449315 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
@@ -26,9 +26,7 @@ DocumentOperation::DocumentOperation(Type type)
}
-DocumentOperation::DocumentOperation(Type type,
- const BucketId &bucketId,
- const Timestamp &timestamp)
+DocumentOperation::DocumentOperation(Type type, const BucketId &bucketId, const Timestamp &timestamp)
: FeedOperation(type),
_bucketId(bucketId),
_timestamp(timestamp),
@@ -62,7 +60,12 @@ vespalib::string DocumentOperation::docArgsToString() const {
}
void
-DocumentOperation::serialize(vespalib::nbostream &os) const
+DocumentOperation::serialize(vespalib::nbostream &os) const {
+ serializeDocumentOperationOnly(os);
+}
+
+void
+DocumentOperation::serializeDocumentOperationOnly(vespalib::nbostream &os) const
{
os << _bucketId;
os << _timestamp;
@@ -74,8 +77,7 @@ DocumentOperation::serialize(vespalib::nbostream &os) const
void
-DocumentOperation::deserialize(vespalib::nbostream &is,
- const DocumentTypeRepo &)
+DocumentOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &)
{
is >> _bucketId;
is >> _timestamp;
@@ -85,4 +87,8 @@ DocumentOperation::deserialize(vespalib::nbostream &is,
is >> _prevTimestamp;
}
+ DbDocumentId DocumentOperation::getDbDocumentId() const {
+ return _dbdId;
+ }
+
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
index bc961c580fc..9a823c553bd 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
@@ -22,124 +22,35 @@ protected:
DocumentOperation(Type type);
- DocumentOperation(Type type,
- const document::BucketId &bucketId,
+ DocumentOperation(Type type, const document::BucketId &bucketId,
const storage::spi::Timestamp &timestamp);
void assertValidBucketId(const document::DocumentId &docId) const;
vespalib::string docArgsToString() const;
public:
- virtual
- ~DocumentOperation()
- {
- }
-
- const
- document::BucketId &
- getBucketId() const
- {
- return _bucketId;
- }
-
- storage::spi::Timestamp
- getTimestamp() const
- {
- return _timestamp;
- }
-
- search::DocumentIdT
- getLid() const
- {
- return _dbdId.getLid();
- }
-
- search::DocumentIdT
- getPrevLid() const
- {
- return _prevDbdId.getLid();
- }
-
- uint32_t
- getSubDbId() const
- {
- return _dbdId.getSubDbId();
- }
-
- uint32_t
- getPrevSubDbId() const
- {
- return _prevDbdId.getSubDbId();
- }
-
- bool
- getValidDbdId() const
- {
- return _dbdId.valid();
- }
-
- bool
- getValidDbdId(uint32_t subDbId) const
- {
- return _dbdId.valid() && _dbdId.getSubDbId() == subDbId;
- }
-
- bool
- getValidPrevDbdId() const
- {
- return _prevDbdId.valid();
- }
-
- bool
- getValidPrevDbdId(uint32_t subDbId) const
- {
- return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId;
- }
-
- bool
- changedDbdId() const
- {
- return _dbdId != _prevDbdId;
- }
- bool
- getPrevMarkedAsRemoved() const
- {
- return _prevMarkedAsRemoved;
- }
-
- void
- setPrevMarkedAsRemoved(bool prevMarkedAsRemoved)
- {
- _prevMarkedAsRemoved = prevMarkedAsRemoved;
- }
-
- DbDocumentId
- getDbDocumentId() const
- {
- return _dbdId;
- }
-
- DbDocumentId
- getPrevDbDocumentId() const
- {
- return _prevDbdId;
- }
-
- void
- setDbDocumentId(DbDocumentId dbdId)
- {
- _dbdId = dbdId;
- }
-
- void
- setPrevDbDocumentId(DbDocumentId prevDbdId)
- {
- _prevDbdId = prevDbdId;
- }
-
- search::DocumentIdT
- getNewOrPrevLid(uint32_t subDbId) const
- {
+ ~DocumentOperation() override {}
+ const document::BucketId &getBucketId() const { return _bucketId; }
+ storage::spi::Timestamp getTimestamp() const { return _timestamp; }
+
+ search::DocumentIdT getLid() const { return _dbdId.getLid(); }
+ search::DocumentIdT getPrevLid() const { return _prevDbdId.getLid(); }
+ uint32_t getSubDbId() const { return _dbdId.getSubDbId(); }
+ uint32_t getPrevSubDbId() const { return _prevDbdId.getSubDbId(); }
+ bool getValidDbdId() const { return _dbdId.valid(); }
+ bool getValidDbdId(uint32_t subDbId) const { return _dbdId.valid() && _dbdId.getSubDbId() == subDbId; }
+ bool getValidPrevDbdId() const { return _prevDbdId.valid(); }
+ bool getValidPrevDbdId(uint32_t subDbId) const { return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId; }
+ bool changedDbdId() const { return _dbdId != _prevDbdId; }
+ bool getPrevMarkedAsRemoved() const { return _prevMarkedAsRemoved; }
+ void setPrevMarkedAsRemoved(bool prevMarkedAsRemoved) { _prevMarkedAsRemoved = prevMarkedAsRemoved; }
+ DbDocumentId getDbDocumentId() const;
+ DbDocumentId getPrevDbDocumentId() const { return _prevDbdId; }
+
+ void setDbDocumentId(DbDocumentId dbdId) { _dbdId = dbdId; }
+ void setPrevDbDocumentId(DbDocumentId prevDbdId) { _prevDbdId = prevDbdId; }
+
+ search::DocumentIdT getNewOrPrevLid(uint32_t subDbId) const {
if (getValidDbdId() && getSubDbId() == subDbId)
return getLid();
if (getValidPrevDbdId() && getPrevSubDbId() == subDbId)
@@ -147,51 +58,34 @@ public:
return 0;
}
- bool
- getValidNewOrPrevDbdId() const
- {
+ bool getValidNewOrPrevDbdId() const {
return getValidDbdId() || getValidPrevDbdId();
}
- bool
- notMovingLidInSameSubDb() const
- {
+ bool notMovingLidInSameSubDb() const {
return !getValidDbdId() ||
!getValidPrevDbdId() ||
getSubDbId() != getPrevSubDbId() ||
getLid() == getPrevLid();
}
- bool
- movingLidIfInSameSubDb() const
- {
+ bool movingLidIfInSameSubDb() const {
return !getValidDbdId() ||
!getValidPrevDbdId() ||
getSubDbId() != getPrevSubDbId() ||
getLid() != getPrevLid();
}
- storage::spi::Timestamp
- getPrevTimestamp() const
- {
- return _prevTimestamp;
- }
-
- void
- setPrevTimestamp(storage::spi::Timestamp prevTimestamp)
- {
- _prevTimestamp = prevTimestamp;
- }
-
- virtual void
- serialize(vespalib::nbostream &os) const override;
+ storage::spi::Timestamp getPrevTimestamp() const { return _prevTimestamp; }
+ void setPrevTimestamp(storage::spi::Timestamp prevTimestamp) { _prevTimestamp = prevTimestamp; }
- virtual void
- deserialize(vespalib::nbostream &is,
- const document::DocumentTypeRepo &repo) override;
+ void serialize(vespalib::nbostream &os) const override;
+ void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override;
uint32_t getSerializedDocSize() const { return _serializedDocSize; }
+
+ // Provided as a hook for tests.
+ void serializeDocumentOperationOnly(vespalib::nbostream &os) const;
};
} // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
index e44b7874c12..8a00c739126 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
@@ -52,8 +52,7 @@ public:
void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; }
SerialNum getSerialNum() const { return _serialNum; }
virtual void serialize(vespalib::nbostream &os) const = 0;
- virtual void deserialize(vespalib::nbostream &is,
- const document::DocumentTypeRepo &repo) = 0;
+ virtual void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) = 0;
virtual vespalib::string toString() const = 0;
};
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp
index beaf719dc5c..3468cc68ced 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp
@@ -28,21 +28,15 @@ UpdateOperation::UpdateOperation(Type type)
}
-UpdateOperation::UpdateOperation(Type type,
- const BucketId &bucketId,
- const Timestamp &timestamp,
- const DocumentUpdate::SP &upd)
- : DocumentOperation(type,
- bucketId,
- timestamp),
+UpdateOperation::UpdateOperation(Type type, const BucketId &bucketId,
+ const Timestamp &timestamp, const DocumentUpdate::SP &upd)
+ : DocumentOperation(type, bucketId, timestamp),
_upd(upd)
{
}
-UpdateOperation::UpdateOperation(const BucketId &bucketId,
- const Timestamp &timestamp,
- const DocumentUpdate::SP &upd)
+UpdateOperation::UpdateOperation(const BucketId &bucketId, const Timestamp &timestamp, const DocumentUpdate::SP &upd)
: UpdateOperation(FeedOperation::UPDATE, bucketId, timestamp, upd)
{
}
@@ -50,20 +44,15 @@ UpdateOperation::UpdateOperation(const BucketId &bucketId,
void
UpdateOperation::serializeUpdate(vespalib::nbostream &os) const
{
- if (getType() == FeedOperation::UPDATE_42) {
- _upd->serialize42(os);
- } else {
- _upd->serializeHEAD(os);
- }
+ assert(getType() == UPDATE);
+ _upd->serializeHEAD(os);
}
void
UpdateOperation::deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo)
{
document::ByteBuffer buf(is.peek(), is.size());
- using Version = DocumentUpdate::SerializeVersion;
- Version version = ((getType() == FeedOperation::UPDATE_42) ? Version::SERIALIZE_42 : Version::SERIALIZE_HEAD);
- DocumentUpdate::SP update(std::make_shared<DocumentUpdate>(repo, buf, version));
+ DocumentUpdate::UP update = (getType() == UPDATE_42) ? DocumentUpdate::create42(repo, buf) : DocumentUpdate::createHEAD(repo, buf);
is.adjustReadPos(buf.getPos());
_upd = std::move(update);
}
@@ -78,8 +67,7 @@ UpdateOperation::serialize(vespalib::nbostream &os) const
void
-UpdateOperation::deserialize(vespalib::nbostream &is,
- const DocumentTypeRepo &repo)
+UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &repo)
{
DocumentOperation::deserialize(is, repo);
try {
@@ -108,12 +96,4 @@ vespalib::string UpdateOperation::toString() const {
docArgsToString().c_str());
}
-UpdateOperation
-UpdateOperation::makeOldUpdate(const document::BucketId &bucketId,
- const storage::spi::Timestamp &timestamp,
- const document::DocumentUpdate::SP &upd)
-{
- return UpdateOperation(FeedOperation::UPDATE_42, bucketId, timestamp, upd);
-}
-
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h
index 7886231af82..99dcbfbce6c 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h
@@ -15,8 +15,7 @@ class UpdateOperation : public DocumentOperation
private:
using DocumentUpdateSP = std::shared_ptr<document::DocumentUpdate>;
DocumentUpdateSP _upd;
- UpdateOperation(Type type,
- const document::BucketId &bucketId,
+ UpdateOperation(Type type, const document::BucketId &bucketId,
const storage::spi::Timestamp &timestamp,
const DocumentUpdateSP &upd);
void serializeUpdate(vespalib::nbostream &os) const;
@@ -27,15 +26,12 @@ public:
UpdateOperation(const document::BucketId &bucketId,
const storage::spi::Timestamp &timestamp,
const DocumentUpdateSP &upd);
- virtual ~UpdateOperation() {}
+ ~UpdateOperation() override {}
const DocumentUpdateSP &getUpdate() const { return _upd; }
void serialize(vespalib::nbostream &os) const override;
void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override;
void deserializeUpdate(const document::DocumentTypeRepo &repo);
- virtual vespalib::string toString() const override;
- static UpdateOperation makeOldUpdate(const document::BucketId &bucketId,
- const storage::spi::Timestamp &timestamp,
- const DocumentUpdateSP &upd);
+ vespalib::string toString() const override;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt
index 78084f29742..0dee7adfa49 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt
@@ -27,6 +27,7 @@ vespa_add_library(searchcore_matching STATIC
ranking_constants.cpp
requestcontext.cpp
result_processor.cpp
+ same_element_builder.cpp
search_session.cpp
session_manager_explorer.cpp
sessionmanager.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp
index 165fc67179a..268fe63ba4c 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp
@@ -2,6 +2,8 @@
#include "querynodes.h"
#include "blueprintbuilder.h"
+#include "termdatafromnode.h"
+#include "same_element_builder.h"
#include <vespa/searchlib/query/tree/customtypevisitor.h>
#include <vespa/searchlib/queryeval/leaf_blueprints.h>
#include <vespa/searchlib/queryeval/intermediate_blueprints.h>
@@ -98,6 +100,15 @@ private:
n.setDocumentFrequency(_result->getState().estimate().estHits, _context.getDocIdLimit());
}
+ void buildSameElement(ProtonSameElement &n) {
+ SameElementBuilder builder(_requestContext, _context);
+ for (size_t i = 0; i < n.getChildren().size(); ++i) {
+ search::query::Node &node = *n.getChildren()[i];
+ builder.add_child(node);
+ }
+ _result = builder.build();
+ }
+
template <typename NodeType>
void buildTerm(NodeType &n) {
FieldSpecList indexFields;
@@ -131,7 +142,7 @@ protected:
void visit(ProtonRank &n) override { buildIntermediate(new RankBlueprint(), n); }
void visit(ProtonNear &n) override { buildIntermediate(new NearBlueprint(n.getDistance()), n); }
void visit(ProtonONear &n) override { buildIntermediate(new ONearBlueprint(n.getDistance()), n); }
- void visit(ProtonSameElement &n) override { buildIntermediate(nullptr /*new SameElementBlueprint())*/, n); }
+ void visit(ProtonSameElement &n) override { buildSameElement(n); }
void visit(ProtonWeightedSetTerm &n) override { buildTerm(n); }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h
index eb45a735780..44cd6ffabfd 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h
@@ -20,4 +20,3 @@ struct BlueprintBuilder {
};
}
-
diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp
new file mode 100644
index 00000000000..d3a0ec4726f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp
@@ -0,0 +1,96 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "same_element_builder.h"
+#include "querynodes.h"
+#include <vespa/searchlib/query/tree/customtypevisitor.h>
+#include <vespa/searchlib/queryeval/leaf_blueprints.h>
+
+using search::queryeval::Blueprint;
+using search::queryeval::EmptyBlueprint;
+using search::queryeval::FieldSpecList;
+using search::queryeval::IRequestContext;
+using search::queryeval::SameElementBlueprint;
+using search::queryeval::Searchable;
+
+namespace proton::matching {
+
+namespace {
+
+class SameElementBuilderVisitor : public search::query::CustomTypeVisitor<ProtonNodeTypes>
+{
+private:
+ const IRequestContext &_requestContext;
+ ISearchContext &_context;
+ SameElementBlueprint &_result;
+
+public:
+ SameElementBuilderVisitor(const IRequestContext &requestContext, ISearchContext &context, SameElementBlueprint &result)
+ : _requestContext(requestContext),
+ _context(context),
+ _result(result) {}
+
+ template <class TermNode>
+ void visitTerm(const TermNode &n) {
+ if (n.numFields() == 1) {
+ const ProtonTermData::FieldEntry &field = n.field(0);
+ assert(field.getFieldId() != search::fef::IllegalFieldId);
+ assert(field.getHandle() == search::fef::IllegalHandle);
+ FieldSpecList field_spec;
+ field_spec.add(_result.getNextChildField(field.field_name, field.getFieldId()));
+ Searchable &searchable = field.attribute_field ? _context.getAttributes() : _context.getIndexes();
+ _result.addTerm(searchable.createBlueprint(_requestContext, field_spec, n));
+ }
+ }
+
+ void visit(ProtonAnd &) override {}
+ void visit(ProtonAndNot &) override {}
+ void visit(ProtonNear &) override {}
+ void visit(ProtonONear &) override {}
+ void visit(ProtonOr &) override {}
+ void visit(ProtonRank &) override {}
+ void visit(ProtonWeakAnd &) override {}
+ void visit(ProtonSameElement &) override {}
+
+ void visit(ProtonWeightedSetTerm &) override {}
+ void visit(ProtonDotProduct &) override {}
+ void visit(ProtonWandTerm &) override {}
+ void visit(ProtonPhrase &) override {}
+ void visit(ProtonEquiv &) override {}
+
+ void visit(ProtonNumberTerm &n) override { visitTerm(n); }
+ void visit(ProtonLocationTerm &n) override { visitTerm(n); }
+ void visit(ProtonPrefixTerm &n) override { visitTerm(n); }
+ void visit(ProtonRangeTerm &n) override { visitTerm(n); }
+ void visit(ProtonStringTerm &n) override { visitTerm(n); }
+ void visit(ProtonSubstringTerm &n) override { visitTerm(n); }
+ void visit(ProtonSuffixTerm &n) override { visitTerm(n); }
+ void visit(ProtonPredicateQuery &) override {}
+ void visit(ProtonRegExpTerm &n) override { visitTerm(n); }
+};
+
+} // namespace proton::matching::<unnamed>
+
+SameElementBuilder::SameElementBuilder(const search::queryeval::IRequestContext &requestContext, ISearchContext &context)
+ : _requestContext(requestContext),
+ _context(context),
+ _result(std::make_unique<SameElementBlueprint>())
+{
+}
+
+void
+SameElementBuilder::add_child(search::query::Node &node)
+{
+ SameElementBuilderVisitor visitor(_requestContext, _context, *_result);
+ node.accept(visitor);
+}
+
+Blueprint::UP
+SameElementBuilder::build()
+{
+ if (!_result || _result->terms().empty()) {
+ return std::make_unique<EmptyBlueprint>();
+ }
+ return std::move(_result);
+}
+
+} // namespace proton::matching
diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h
new file mode 100644
index 00000000000..945bb9a97f6
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h
@@ -0,0 +1,24 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "isearchcontext.h"
+#include <vespa/searchlib/query/tree/node.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/queryeval/same_element_blueprint.h>
+
+namespace proton::matching {
+
+class SameElementBuilder
+{
+private:
+ const search::queryeval::IRequestContext &_requestContext;
+ ISearchContext &_context;
+ std::unique_ptr<search::queryeval::SameElementBlueprint> _result;
+public:
+ SameElementBuilder(const search::queryeval::IRequestContext &requestContext, ISearchContext &context);
+ void add_child(search::query::Node &node);
+ search::queryeval::Blueprint::UP build();
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
index 2b2849d025c..78733b14aaa 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
@@ -13,23 +13,6 @@ using search::index::Schema;
namespace proton {
-FastAccessFeedView::UpdateScope
-FastAccessFeedView::getUpdateScope(const DocumentUpdate &upd)
-{
- UpdateScope updateScope;
- for (const auto &update : upd.getUpdates()) {
- const vespalib::string &fieldName = update.getField().getName();
- if (!fastPartialUpdateAttribute(fieldName)) {
- updateScope._nonAttributeFields = true;
- break;
- }
- }
- if (!upd.getFieldPathUpdates().empty()) {
- updateScope._nonAttributeFields = true;
- }
- return updateScope;
-}
-
/**
* NOTE: For both put, update and remove we only need to pass the 'onWriteDone'
* instance when we are going to commit as part of handling the operation.
@@ -47,9 +30,18 @@ FastAccessFeedView::putAttributes(SerialNum serialNum, search::DocumentIdT lid,
void
FastAccessFeedView::updateAttributes(SerialNum serialNum, search::DocumentIdT lid, const DocumentUpdate &upd,
+ bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate)
+{
+ _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone, onUpdate);
+}
+
+void
+FastAccessFeedView::updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc,
bool immediateCommit, OnOperationDoneType onWriteDone)
{
- _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone);
+ if (_attributeWriter->getHasCompoundAttribute()) {
+ _attributeWriter->update(serialNum, *doc.get(), lid, immediateCommit, onWriteDone);
+ }
}
void
@@ -107,19 +99,4 @@ FastAccessFeedView::sync()
_writeService.attributeFieldWriter().sync();
}
-bool
-FastAccessFeedView::fastPartialUpdateAttribute(const vespalib::string &fieldName) const {
- search::AttributeVector *attribute = _attributeWriter->getWritableAttribute(fieldName);
- if (attribute == nullptr) {
- // Partial update to non-attribute field must update document
- return false;
- }
- search::attribute::BasicType::Type attrType = attribute->getBasicType();
- // Partial update to tensor, predicate or reference attribute
- // must update document
- return ((attrType != search::attribute::BasicType::Type::PREDICATE) &&
- (attrType != search::attribute::BasicType::Type::TENSOR) &&
- (attrType != search::attribute::BasicType::Type::REFERENCE));
-}
-
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h
index e1b0cf83f64..3af97b4ecb9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h
@@ -39,14 +39,13 @@ private:
const IAttributeWriter::SP _attributeWriter;
DocIdLimit &_docIdLimit;
- UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override;
-
void putAttributes(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc,
bool immediateCommit, OnPutDoneType onWriteDone) override;
void updateAttributes(SerialNum serialNum, search::DocumentIdT lid, const document::DocumentUpdate &upd,
+ bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate) override;
+ void updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc,
bool immediateCommit, OnOperationDoneType onWriteDone) override;
-
void removeAttributes(SerialNum serialNum, search::DocumentIdT lid,
bool immediateCommit, OnRemoveDoneType onWriteDone) override;
@@ -73,8 +72,6 @@ public:
void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
void sync() override;
-
- bool fastPartialUpdateAttribute(const vespalib::string &fieldName) const;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
index 0c87d24899d..4cda07eee8b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
@@ -100,7 +100,7 @@ void
SearchableFeedView::performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc futureDoc,
bool immediateCommit, OnOperationDoneType onWriteDone)
{
- Document::UP doc = std::move(futureDoc.get());
+ const auto &doc = futureDoc.get();
if (doc) {
performIndexPut(serialNum, lid, *doc, immediateCommit, onWriteDone);
}
@@ -118,29 +118,6 @@ SearchableFeedView::performIndexHeartBeat(SerialNum serialNum)
_indexWriter->heartBeat(serialNum);
}
-SearchableFeedView::UpdateScope
-SearchableFeedView::getUpdateScope(const DocumentUpdate &upd)
-{
- UpdateScope updateScope;
- const Schema &schema = *_schema;
- for(size_t i(0), m(upd.getUpdates().size());
- !(updateScope._indexedFields && updateScope._nonAttributeFields) &&
- (i < m); i++) {
- const document::FieldUpdate & fu(upd.getUpdates()[i]);
- const vespalib::string &name = fu.getField().getName();
- if (schema.isIndexField(name)) {
- updateScope._indexedFields = true;
- }
- if (!fastPartialUpdateAttribute(name)) {
- updateScope._nonAttributeFields = true;
- }
- }
- if (!upd.getFieldPathUpdates().empty()) {
- updateScope._nonAttributeFields = true;
- }
- return updateScope;
-}
-
void
SearchableFeedView::updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc futureDoc,
bool immediateCommit, OnOperationDoneType onWriteDone)
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
index 81d81a6cc27..ed3ba6740b1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
@@ -34,24 +34,19 @@ private:
bool hasIndexedFields() const { return _hasIndexedFields; }
- void
- performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc,
- bool immediateCommit, OnOperationDoneType onWriteDone);
+ void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc,
+ bool immediateCommit, OnOperationDoneType onWriteDone);
- void
- performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &doc,
- bool immediateCommit, OnOperationDoneType onWriteDone);
- void
- performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc doc,
- bool immediateCommit, OnOperationDoneType onWriteDone);
+ void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &doc,
+ bool immediateCommit, OnOperationDoneType onWriteDone);
+ void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc doc,
+ bool immediateCommit, OnOperationDoneType onWriteDone);
- void
- performIndexRemove(SerialNum serialNum, search::DocumentIdT lid,
- bool immediateCommit, OnRemoveDoneType onWriteDone);
+ void performIndexRemove(SerialNum serialNum, search::DocumentIdT lid,
+ bool immediateCommit, OnRemoveDoneType onWriteDone);
- void
- performIndexRemove(SerialNum serialNum, const LidVector &lidsToRemove,
- bool immediateCommit, OnWriteDoneType onWriteDone);
+ void performIndexRemove(SerialNum serialNum, const LidVector &lidsToRemove,
+ bool immediateCommit, OnWriteDoneType onWriteDone);
void performIndexHeartBeat(SerialNum serialNum);
@@ -60,23 +55,17 @@ private:
void performSync();
void heartBeatIndexedFields(SerialNum serialNum) override;
- virtual void
- putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &newDoc,
- bool immediateCommit, OnOperationDoneType onWriteDone) override;
+ void putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &newDoc,
+ bool immediateCommit, OnOperationDoneType onWriteDone) override;
- UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override;
+ void updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc newDoc,
+ bool immediateCommit, OnOperationDoneType onWriteDone) override;
- virtual void
- updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc newDoc,
- bool immediateCommit, OnOperationDoneType onWriteDone) override;
+ void removeIndexedFields(SerialNum serialNum, search::DocumentIdT lid,
+ bool immediateCommit, OnRemoveDoneType onWriteDone) override;
- virtual void
- removeIndexedFields(SerialNum serialNum, search::DocumentIdT lid,
- bool immediateCommit, OnRemoveDoneType onWriteDone) override;
-
- virtual void
- removeIndexedFields(SerialNum serialNum, const LidVector &lidsToRemove,
- bool immediateCommit, OnWriteDoneType onWriteDone) override;
+ void removeIndexedFields(SerialNum serialNum, const LidVector &lidsToRemove,
+ bool immediateCommit, OnWriteDoneType onWriteDone) override;
void performIndexForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone);
void forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) override;
@@ -85,7 +74,7 @@ public:
SearchableFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx, const PersistentParams &params,
const FastAccessFeedView::Context &fastUpdateCtx, Context ctx);
- virtual ~SearchableFeedView();
+ ~SearchableFeedView() override;
const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; }
void sync() override;
};
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index fdaf07dc466..a0f6ee98b71 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -17,6 +17,8 @@
#include <vespa/vespalib/util/exceptions.h>
#include <vespa/log/log.h>
+#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h>
+
LOG_SETUP(".proton.server.storeonlyfeedview");
using document::BucketId;
@@ -300,20 +302,19 @@ StoreOnlyFeedView::heartBeatIndexedFields(SerialNum ) {}
void
StoreOnlyFeedView::heartBeatAttributes(SerialNum ) {}
-
-StoreOnlyFeedView::UpdateScope
-StoreOnlyFeedView::getUpdateScope(const DocumentUpdate &upd)
+void
+StoreOnlyFeedView::updateAttributes(SerialNum, Lid, const DocumentUpdate & upd, bool,
+ OnOperationDoneType, IFieldUpdateCallback & onUpdate)
{
- UpdateScope updateScope;
- if (!upd.getUpdates().empty() || !upd.getFieldPathUpdates().empty()) {
- updateScope._nonAttributeFields = true;
+ for (const auto & fieldUpdate : upd.getUpdates()) {
+ onUpdate.onUpdateField(fieldUpdate.getField().getName(), nullptr);
}
- return updateScope;
}
-
void
-StoreOnlyFeedView::updateAttributes(SerialNum, Lid, const DocumentUpdate &, bool, OnOperationDoneType) {}
+StoreOnlyFeedView::updateAttributes(SerialNum, Lid, FutureDoc, bool, OnOperationDoneType)
+{
+}
void
StoreOnlyFeedView::updateIndexedFields(SerialNum, Lid, FutureDoc, bool, OnOperationDoneType)
@@ -385,6 +386,34 @@ void StoreOnlyFeedView::heartBeatSummary(SerialNum serialNum) {
}));
}
+StoreOnlyFeedView::UpdateScope::UpdateScope(const search::index::Schema & schema, const DocumentUpdate & upd)
+ : _schema(&schema),
+ _indexedFields(false),
+ _nonAttributeFields(!upd.getFieldPathUpdates().empty())
+{}
+
+namespace {
+
+bool isAttributeUpdateable(const search::AttributeVector *attribute) {
+ search::attribute::BasicType::Type attrType = attribute->getBasicType();
+ // Partial update to tensor, predicate or reference attribute
+ // must update document
+ return ((attrType != search::attribute::BasicType::Type::PREDICATE) &&
+ (attrType != search::attribute::BasicType::Type::TENSOR) &&
+ (attrType != search::attribute::BasicType::Type::REFERENCE));
+}
+}
+
+void
+StoreOnlyFeedView::UpdateScope::onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) {
+ if (!_nonAttributeFields && (attr == nullptr || !isAttributeUpdateable(attr))) {
+ _nonAttributeFields = true;
+ }
+ if (!_indexedFields && _schema->isIndexField(fieldName)) {
+ _indexedFields = true;
+ }
+}
+
void
StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) {
if ( ! updOp.getUpdate()) {
@@ -417,15 +446,15 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp)
bool immediateCommit = _commitTimeTracker.needCommit();
auto onWriteDone = createUpdateDoneContext(std::move(token), updOp.getUpdate());
- updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone);
+ UpdateScope updateScope(*_schema, upd);
+ updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope);
- UpdateScope updateScope(getUpdateScope(upd));
if (updateScope.hasIndexOrNonAttributeFields()) {
PromisedDoc promisedDoc;
- FutureDoc futureDoc = promisedDoc.get_future();
+ FutureDoc futureDoc = promisedDoc.get_future().share();
_pendingLidTracker.waitForConsumedLid(lid);
if (updateScope._indexedFields) {
- updateIndexedFields(serialNum, lid, std::move(futureDoc), immediateCommit, onWriteDone);
+ updateIndexedFields(serialNum, lid, futureDoc, immediateCommit, onWriteDone);
}
PromisedStream promisedStream;
FutureStream futureStream = promisedStream.get_future();
@@ -444,6 +473,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp)
std::move(promisedDoc), std::move(promisedStream));
});
#pragma GCC diagnostic pop
+ updateAttributes(serialNum, lid, std::move(futureDoc), immediateCommit, onWriteDone);
}
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
index b106b87c4fe..a11512590f3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
@@ -9,6 +9,7 @@
#include "searchcontext.h"
#include "pendinglidtracker.h"
#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h>
#include <vespa/searchcore/proton/common/feeddebugger.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
@@ -33,6 +34,7 @@ class PutDoneContext;
class RemoveDoneContext;
class CommitTimeTracker;
class IGidToLidChangeHandler;
+class IFieldUpdateCallback;
namespace documentmetastore { class ILidReuseDelayer; }
@@ -59,7 +61,7 @@ public:
using OnPutDoneType = const std::shared_ptr<PutDoneContext> &;
using OnRemoveDoneType = const std::shared_ptr<RemoveDoneContext> &;
using FeedTokenUP = std::unique_ptr<FeedToken>;
- using FutureDoc = std::future<std::unique_ptr<Document>>;
+ using FutureDoc = std::shared_future<std::unique_ptr<Document>>;
using PromisedDoc = std::promise<std::unique_ptr<Document>>;
using FutureStream = std::future<vespalib::nbostream>;
using PromisedStream = std::promise<vespalib::nbostream>;
@@ -120,18 +122,19 @@ public:
};
protected:
- struct UpdateScope
+ class UpdateScope : public IFieldUpdateCallback
{
+ private:
+ const search::index::Schema *_schema;
+ public:
bool _indexedFields;
bool _nonAttributeFields;
- UpdateScope()
- : _indexedFields(false),
- _nonAttributeFields(false)
- {}
+ UpdateScope(const search::index::Schema & schema, const DocumentUpdate & upd);
bool hasIndexOrNonAttributeFields() const {
return _indexedFields || _nonAttributeFields;
}
+ void onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) override;
};
private:
@@ -200,9 +203,10 @@ private:
virtual void putIndexedFields(SerialNum serialNum, Lid lid, const DocumentSP &newDoc,
bool immediateCommit, OnOperationDoneType onWriteDone);
- virtual UpdateScope getUpdateScope(const DocumentUpdate &upd);
-
virtual void updateAttributes(SerialNum serialNum, Lid lid, const DocumentUpdate &upd,
+ bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate);
+
+ virtual void updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc,
bool immediateCommit, OnOperationDoneType onWriteDone);
virtual void updateIndexedFields(SerialNum serialNum, Lid lid, FutureDoc doc,
diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h
index e0fa06c5e84..a524f2ce0fa 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h
@@ -83,6 +83,8 @@ protected:
public:
AttributeIteratorT(const SC &searchContext, fef::TermFieldMatchData *matchData);
bool seekFast(uint32_t docId) const { return _searchContext.matches(docId); }
+
+ const attribute::ISearchContext * getAttributeSearchContext() const override { return &_searchContext; }
};
template <typename SC>
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
index 4dbcc85d861..1ccc9923c99 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
@@ -73,7 +73,7 @@ AttributeVector::BaseName::BaseName(const vespalib::stringref &base,
append(name);
}
-AttributeVector::BaseName::~BaseName() { }
+AttributeVector::BaseName::~BaseName() = default;
AttributeVector::BaseName::string
@@ -165,9 +165,7 @@ AttributeVector::AttributeVector(const vespalib::stringref &baseFileName, const
_genHandler(),
_genHolder(),
_status(Status::createName((_baseFileName.getIndexName() +
- (_baseFileName.getSnapshotName().empty() ?
- "" :
- ".") +
+ (_baseFileName.getSnapshotName().empty() ? "" : ".") +
_baseFileName.getSnapshotName()),
_baseFileName.getAttributeName())),
_highestValueCount(1),
@@ -177,29 +175,24 @@ AttributeVector::AttributeVector(const vespalib::stringref &baseFileName, const
_createSerialNum(0u),
_compactLidSpaceGeneration(0u),
_hasEnum(false),
- _hasSortedEnum(false),
_loaded(false),
_enableEnumeratedSave(false)
{ }
-
-AttributeVector::~AttributeVector() { }
+AttributeVector::~AttributeVector() = default;
void AttributeVector::updateStat(bool force) {
if (force) {
onUpdateStat();
} else if (_nextStatUpdateTime < fastos::ClockSystem::now()) {
onUpdateStat();
- _nextStatUpdateTime = fastos::ClockSystem::now() +
- fastos::TimeStamp::SEC;
+ _nextStatUpdateTime = fastos::ClockSystem::now() + fastos::TimeStamp::SEC;
}
}
-size_t AttributeVector::getFixedWidth() const { return _config.basicType().fixedSize(); }
bool AttributeVector::hasEnum() const { return _hasEnum; }
bool AttributeVector::hasEnum2Value() const { return false; }
uint32_t AttributeVector::getMaxValueCount() const { return _highestValueCount; }
-uint32_t AttributeVector::getNumDocs() const { return _status.getNumDocs(); }
bool
AttributeVector::isEnumerated(const vespalib::GenericHeader &header)
@@ -217,13 +210,11 @@ AttributeVector::commit(bool forceUpdateStat)
_loaded = true;
}
-
void
AttributeVector::commit(uint64_t firstSyncToken, uint64_t lastSyncToken)
{
if (firstSyncToken < getStatus().getLastSyncToken()) {
- LOG(error,
- "Expected first token to be >= %" PRIu64 ", got %" PRIu64 ".",
+ LOG(error, "Expected first token to be >= %" PRIu64 ", got %" PRIu64 ".",
getStatus().getLastSyncToken(), firstSyncToken);
abort();
}
@@ -231,7 +222,6 @@ AttributeVector::commit(uint64_t firstSyncToken, uint64_t lastSyncToken)
_status.setLastSyncToken(lastSyncToken);
}
-
bool
AttributeVector::addDocs(DocId &startDoc, DocId &lastDoc, uint32_t numDocs)
{
@@ -271,19 +261,10 @@ AttributeVector::incGeneration()
void
-AttributeVector::updateStatistics(uint64_t numValues,
- uint64_t numUniqueValue,
- uint64_t allocated,
- uint64_t used,
- uint64_t dead,
- uint64_t onHold)
+AttributeVector::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated,
+ uint64_t used, uint64_t dead, uint64_t onHold)
{
- _status.updateStatistics(numValues,
- numUniqueValue,
- allocated,
- used,
- dead,
- onHold);
+ _status.updateStatistics(numValues, numUniqueValue, allocated, used, dead, onHold);
}
AddressSpace
@@ -292,16 +273,6 @@ AttributeVector::getEnumStoreAddressSpaceUsage() const
return AddressSpaceUsage::defaultEnumStoreUsage();
}
-bool
-AttributeVector::hasMultiValue() const {
- return _config.collectionType().isMultiValue();
-}
-
-bool
-AttributeVector::hasWeightedSetType() const {
- return _config.collectionType().isWeightedSet();
-}
-
AddressSpace
AttributeVector::getMultiValueAddressSpaceUsage() const
{
@@ -311,38 +282,7 @@ AttributeVector::getMultiValueAddressSpaceUsage() const
AddressSpaceUsage
AttributeVector::getAddressSpaceUsage() const
{
- return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(),
- getMultiValueAddressSpaceUsage());
-}
-
-const vespalib::string &
-AttributeVector::getName() const {
- return _baseFileName.getAttributeName();
-}
-
-attribute::BasicType::Type
-AttributeVector::getBasicType() const {
- return getInternalBasicType().type();
-}
-attribute::CollectionType::Type
-AttributeVector::getCollectionType() const {
- return getInternalCollectionType().type();
-}
-
-bool
-AttributeVector::getIsFilter() const {
- return _config.getIsFilter();
-}
-
-bool
-AttributeVector::getIsFastSearch() const {
- return _config.fastSearch();
-}
-
-uint32_t
-AttributeVector::getCommittedDocIdLimit() const
-{
- return _committedDocIdLimit;
+ return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), getMultiValueAddressSpaceUsage());
}
bool
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h
index b25c7b67299..87ef9a41432 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h
@@ -213,7 +213,6 @@ protected:
void setEnumMax(uint32_t e) { _enumMax = e; setEnum(); }
void setEnum(bool hasEnum_=true) { _hasEnum = hasEnum_; }
- void setSortedEnum(bool sorted=true) { _hasSortedEnum = sorted; }
void setNumDocs(uint32_t n) { _status.setNumDocs(n); }
void incNumDocs() { _status.incNumDocs(); }
@@ -398,7 +397,7 @@ public:
bool isLoaded() const { return _loaded; }
/** Return the fixed length of the attribute. If 0 then you must inquire each document. */
- virtual size_t getFixedWidth() const override;
+ size_t getFixedWidth() const override { return _config.basicType().fixedSize(); }
const Config &getConfig() const { return _config; }
BasicType getInternalBasicType() const { return _config.basicType(); }
CollectionType getInternalCollectionType() const { return _config.collectionType(); }
@@ -406,17 +405,16 @@ public:
void setBaseFileName(const vespalib::stringref & name) { _baseFileName = name; }
// Implements IAttributeVector
- virtual const vespalib::string & getName() const override;
+ const vespalib::string & getName() const override final { return _baseFileName.getAttributeName(); }
bool hasArrayType() const { return _config.collectionType().isArray(); }
- bool hasEnum() const override;
- bool hasSortedEnum() const { return _hasSortedEnum; }
+ bool hasEnum() const override final;
virtual bool hasEnum2Value() const;
uint32_t getMaxValueCount() const override;
uint32_t getEnumMax() const { return _enumMax; }
// Implements IAttributeVector
- uint32_t getNumDocs() const override;
+ uint32_t getNumDocs() const override final { return _status.getNumDocs(); }
uint32_t & getCommittedDocIdLimitRef() { return _committedDocIdLimit; }
void setCommittedDocIdLimit(uint32_t committedDocIdLimit) {
_committedDocIdLimit = committedDocIdLimit;
@@ -427,13 +425,12 @@ public:
AddressSpaceUsage getAddressSpaceUsage() const;
- // Implements IAttributeVector
- virtual BasicType::Type getBasicType() const override;
- virtual CollectionType::Type getCollectionType() const override;
- virtual bool getIsFilter() const override;
- virtual bool getIsFastSearch() const override;
- virtual uint32_t getCommittedDocIdLimit() const override;
- virtual bool isImported() const override;
+ BasicType::Type getBasicType() const override final { return getInternalBasicType().type(); }
+ CollectionType::Type getCollectionType() const override final { return getInternalCollectionType().type(); }
+ bool getIsFilter() const override final { return _config.getIsFilter(); }
+ bool getIsFastSearch() const override final { return _config.fastSearch(); }
+ uint32_t getCommittedDocIdLimit() const override final { return _committedDocIdLimit; }
+ bool isImported() const override;
/**
* Updates the base file name of this attribute vector and saves
@@ -604,7 +601,6 @@ private:
uint64_t _createSerialNum;
uint64_t _compactLidSpaceGeneration;
bool _hasEnum;
- bool _hasSortedEnum;
bool _loaded;
bool _enableEnumeratedSave;
fastos::TimeStamp _nextStatUpdateTime;
@@ -634,8 +630,8 @@ private:
friend class AttributeManagerTest;
public:
bool headerTypeOK(const vespalib::GenericHeader &header) const;
- bool hasMultiValue() const override;
- bool hasWeightedSetType() const override;
+ bool hasMultiValue() const override final { return _config.collectionType().isMultiValue(); }
+ bool hasWeightedSetType() const override final { return _config.collectionType().isWeightedSet(); }
/**
* Should be called by the writer thread.
*/
diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp
index abcdc0244c2..8f67d237a60 100644
--- a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp
@@ -170,7 +170,6 @@ StringDirectAttrVector(const vespalib::string & baseFileName, const Config & c)
_idx.push_back(0);
}
setEnum();
- setSortedEnum(true);
}
template <typename F>
@@ -182,6 +181,5 @@ StringDirectAttrVector(const vespalib::string & baseFileName) :
_idx.push_back(0);
}
setEnum();
- setSortedEnum(true);
}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp
index d4fa47eac49..ee984688594 100644
--- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp
@@ -12,7 +12,6 @@ SingleStringExtAttribute::SingleStringExtAttribute(const vespalib::string & name
StringDirectAttrVector< AttrVector::Features<false> >(name, Config(BasicType::STRING, CollectionType::SINGLE))
{
setEnum(false);
- setSortedEnum(false);
}
bool SingleStringExtAttribute::addDoc(DocId & docId)
@@ -45,7 +44,6 @@ MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name,
(name, Config(BasicType::STRING, ctype))
{
setEnum(false);
- setSortedEnum(false);
}
MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name) :
@@ -53,7 +51,6 @@ MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name)
(name, Config(BasicType::STRING, CollectionType::ARRAY))
{
setEnum(false);
- setSortedEnum(false);
}
bool MultiStringExtAttribute::addDoc(DocId & docId)
@@ -138,7 +135,6 @@ WeightedSetStringExtAttribute::WeightedSetStringExtAttribute(const vespalib::str
WeightedSetExtAttributeBase<MultiStringExtAttribute>(name)
{
setEnum(false);
- setSortedEnum(false);
}
WeightedSetStringExtAttribute::~WeightedSetStringExtAttribute() {}
diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp
index a563a288396..840bb1e5ea9 100644
--- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp
@@ -3,6 +3,7 @@
#include "same_element_blueprint.h"
#include "same_element_search.h"
#include <vespa/searchlib/fef/termfieldmatchdata.h>
+#include <vespa/searchlib/attribute/elementiterator.h>
#include <vespa/vespalib/objects/visit.hpp>
#include <algorithm>
#include <map>
@@ -60,14 +61,29 @@ SameElementBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArra
{
(void) tfmda;
assert(!tfmda.valid());
- fef::MatchData::UP md = _layout.createMatchData();
+
+ fef::MatchDataLayout my_layout = _layout;
+ std::vector<fef::TermFieldHandle> extra_handles;
+ for (size_t i = 0; i < _terms.size(); ++i) {
+ const State &childState = _terms[i]->getState();
+ assert(childState.numFields() == 1);
+ extra_handles.push_back(my_layout.allocTermField(childState.field(0).getFieldId()));
+ }
+ fef::MatchData::UP md = my_layout.createMatchData();
search::fef::TermFieldMatchDataArray childMatch;
std::vector<SearchIterator::UP> children(_terms.size());
for (size_t i = 0; i < _terms.size(); ++i) {
const State &childState = _terms[i]->getState();
- assert(childState.numFields() == 1);
- childMatch.add(childState.field(0).resolve(*md));
- children[i] = _terms[i]->createSearch(*md, (strict && (i == 0)));
+ SearchIterator::UP child = _terms[i]->createSearch(*md, (strict && (i == 0)));
+ const attribute::ISearchContext *context = child->getAttributeSearchContext();
+ if (context == nullptr) {
+ children[i] = std::move(child);
+ childMatch.add(childState.field(0).resolve(*md));
+ } else {
+ fef::TermFieldMatchData *child_tfmd = md->resolveTermField(extra_handles[i]);
+ children[i] = std::make_unique<attribute::ElementIterator>(std::move(child), *context, *child_tfmd);
+ childMatch.add(child_tfmd);
+ }
}
return std::make_unique<SameElementSearch>(std::move(md), std::move(children), childMatch, strict);
}
diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp
index 1f5d090b914..3384e0fc8c8 100644
--- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp
@@ -118,7 +118,13 @@ SearchIterator::visitMembers(vespalib::ObjectVisitor &visitor) const
visit(visitor, "docid", _docid);
visit(visitor, "endid", _endid);
}
-
+
+const attribute::ISearchContext *
+SearchIterator::getAttributeSearchContext() const
+{
+ return nullptr;
+}
+
} // namespace queryeval
} // namespace search
diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h
index 63c2afd33aa..dfa342b018a 100644
--- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h
+++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h
@@ -12,6 +12,7 @@
namespace vespalib { class ObjectVisitor; }
namespace search { class BitVector; }
+namespace search::attribute { class ISearchContext; }
namespace search::queryeval {
@@ -338,6 +339,8 @@ public:
virtual Trinary is_strict() const { return Trinary::Undefined; }
+ /** return the underlying attribute search context (or null if none available) */
+ virtual const attribute::ISearchContext *getAttributeSearchContext() const;
};
}
diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml
index 0d66951f364..f31072aec6e 100644
--- a/standalone-container/pom.xml
+++ b/standalone-container/pom.xml
@@ -69,11 +69,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-xml_${scala.major-version}</artifactId>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
@@ -101,40 +96,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
-
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>compile</id>
- <goals>
- <goal>compile</goal>
- </goals>
- <phase>compile</phase>
- </execution>
- <execution>
- <id>test-compile</id>
- <goals>
- <goal>testCompile</goal>
- </goals>
- <phase>test-compile</phase>
- </execution>
- <execution>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <phase>process-test-resources</phase>
- <id>early-test-compile</id>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</project>
diff --git a/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java b/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java
new file mode 100644
index 00000000000..8d4126a01e3
--- /dev/null
+++ b/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java
@@ -0,0 +1,569 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.application.container.impl;
+
+import com.google.common.collect.Lists;
+import com.yahoo.container.standalone.StandaloneContainerApplication;
+import com.yahoo.jdisc.application.OsgiFramework;
+import com.yahoo.jdisc.application.OsgiHeader;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.BundleListener;
+import org.osgi.framework.Filter;
+import org.osgi.framework.FrameworkListener;
+import org.osgi.framework.ServiceFactory;
+import org.osgi.framework.ServiceListener;
+import org.osgi.framework.ServiceObjects;
+import org.osgi.framework.ServiceReference;
+import org.osgi.framework.ServiceRegistration;
+import org.osgi.framework.Version;
+import org.osgi.framework.wiring.BundleCapability;
+import org.osgi.framework.wiring.BundleRequirement;
+import org.osgi.framework.wiring.BundleRevision;
+import org.osgi.framework.wiring.BundleWire;
+import org.osgi.framework.wiring.BundleWiring;
+import org.osgi.resource.Capability;
+import org.osgi.resource.Requirement;
+import org.osgi.resource.Wire;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.Enumeration;
+import java.util.Hashtable;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
+
+/**
+ * A (mock) OSGI implementation which loads classes from the system classpath
+ *
+ * @author Tony Vaagenes
+ * @author ollivir
+ */
+public final class ClassLoaderOsgiFramework implements OsgiFramework {
+ private BundleContextImpl bundleContextImpl = new BundleContextImpl();
+ private SystemBundleImpl systemBundleImpl = new SystemBundleImpl();
+ private BundleWiringImpl bundleWiringImpl = new BundleWiringImpl();
+
+ private List<URL> bundleLocations = new ArrayList<>();
+ private List<Bundle> bundleList = Lists.newArrayList(systemBundleImpl);
+ private ClassLoader classLoader = null;
+
+ private AtomicInteger nextBundleId = new AtomicInteger(1);
+
+ @Override
+ public List<Bundle> installBundle(String bundleLocation) {
+ if (bundleLocation != null && bundleLocation.isEmpty() == false) {
+ try {
+ URL url = new URL(bundleLocation);
+ bundleLocations.add(url);
+ bundleList.add(new JarBundleImpl(url));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ return bundles();
+ }
+
+ private ClassLoader getClassLoader() {
+ if (bundleLocations.isEmpty()) {
+ return getClass().getClassLoader();
+ } else {
+ if (classLoader == null) {
+ classLoader = new URLClassLoader(bundleLocations.toArray(new URL[0]), getClass().getClassLoader());
+ }
+ return classLoader;
+ }
+ }
+
+ @Override
+ public void startBundles(List<Bundle> bundles, boolean privileged) {
+ }
+
+ @Override
+ public void refreshPackages() {
+ }
+
+ @Override
+ public BundleContext bundleContext() {
+ return bundleContextImpl;
+ }
+
+ @Override
+ public List<Bundle> bundles() {
+ return bundleList;
+ }
+
+ @Override
+ public void start() {
+ }
+
+ @Override
+ public void stop() {
+ }
+
+ private abstract class BundleImpl implements Bundle {
+ @Override
+ public int getState() {
+ return Bundle.ACTIVE;
+ }
+
+ @Override
+ public void start(int options) {
+ }
+
+ @Override
+ public void start() {
+ }
+
+ @Override
+ public void stop(int options) {
+ }
+
+ @Override
+ public void stop() {
+ }
+
+ @Override
+ public void update(InputStream input) {
+ }
+
+ @Override
+ public void update() {
+ }
+
+ @Override
+ public void uninstall() {
+ }
+
+ @Override
+ public Dictionary<String, String> getHeaders(String locale) {
+ return getHeaders();
+ }
+
+ @Override
+ public String getSymbolicName() {
+ return ClassLoaderOsgiFramework.this.getClass().getName();
+ }
+
+ @Override
+ public String getLocation() {
+ return getSymbolicName();
+ }
+
+ @Override
+ public ServiceReference<?>[] getRegisteredServices() {
+ return new ServiceReference<?>[0];
+ }
+
+ @Override
+ public ServiceReference<?>[] getServicesInUse() {
+ return getRegisteredServices();
+ }
+
+ @Override
+ public boolean hasPermission(Object permission) {
+ return true;
+ }
+
+ @Override
+ public URL getResource(String name) {
+ return getClassLoader().getResource(name);
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ return getClassLoader().loadClass(name);
+ }
+
+ @Override
+ public Enumeration<URL> getResources(String name) throws IOException {
+ return getClassLoader().getResources(name);
+ }
+
+ @Override
+ public Enumeration<String> getEntryPaths(String path) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public URL getEntry(String path) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Enumeration<URL> findEntries(String path, String filePattern, boolean recurse) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getLastModified() {
+ return 1L;
+ }
+
+ @Override
+ public BundleContext getBundleContext() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Map<X509Certificate, List<X509Certificate>> getSignerCertificates(int signersType) {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> T adapt(Class<T> clazz) {
+ if (clazz.equals(BundleRevision.class)) {
+ return (T) new BundleRevisionImpl();
+ } else if (clazz.equals(BundleWiring.class)) {
+ return (T) new BundleWiringImpl();
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public File getDataFile(String filename) {
+ return null;
+ }
+
+ @Override
+ public int compareTo(Bundle o) {
+ return Long.compare(getBundleId(), o.getBundleId());
+ }
+ }
+
+ private class BundleRevisionImpl implements BundleRevision {
+ @Override
+ public String getSymbolicName() {
+ return this.getClass().getName();
+ }
+
+ @Override
+ public List<BundleRequirement> getDeclaredRequirements(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Version getVersion() {
+ return Version.emptyVersion;
+ }
+
+ @Override
+ public BundleWiring getWiring() {
+ return bundleWiringImpl;
+ }
+
+ @Override
+ public List<BundleCapability> getDeclaredCapabilities(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getTypes() {
+ return 0;
+ }
+
+ @Override
+ public Bundle getBundle() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<Capability> getCapabilities(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<Requirement> getRequirements(String p1) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private class BundleWiringImpl implements BundleWiring {
+ @Override
+ public List<URL> findEntries(String p1, String p2, int p3) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<Wire> getRequiredResourceWires(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<Capability> getResourceCapabilities(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isCurrent() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<BundleWire> getRequiredWires(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<BundleCapability> getCapabilities(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<Wire> getProvidedResourceWires(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<BundleWire> getProvidedWires(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BundleRevision getRevision() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<Requirement> getResourceRequirements(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isInUse() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Collection<String> listResources(String p1, String p2, int p3) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ClassLoader getClassLoader() {
+ return ClassLoaderOsgiFramework.this.getClassLoader();
+ }
+
+ @Override
+ public List<BundleRequirement> getRequirements(String p1) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BundleRevision getResource() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Bundle getBundle() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private class SystemBundleImpl extends BundleImpl {
+ @Override
+ public long getBundleId() {
+ return 0L;
+ }
+
+ @Override
+ public Version getVersion() {
+ return Version.emptyVersion;
+ }
+
+ @Override
+ public Dictionary<String, String> getHeaders() {
+ Hashtable<String, String> ret = new Hashtable<>();
+ ret.put(OsgiHeader.APPLICATION, StandaloneContainerApplication.class.getName());
+ return ret;
+ }
+ }
+
+ private class JarBundleImpl extends BundleImpl {
+ private final long bundleId;
+ private final Dictionary<String, String> headers;
+
+ JarBundleImpl(URL location) throws IOException {
+ this.bundleId = (long) nextBundleId.getAndIncrement();
+ this.headers = retrieveHeaders(location);
+ }
+
+ @Override
+ public long getBundleId() {
+ return bundleId;
+ }
+
+ @Override
+ public Dictionary<String, String> getHeaders() {
+ return headers;
+ }
+
+ @Override
+ public String getSymbolicName() {
+ return headers.get("Bundle-SymbolicName");
+ }
+
+ @Override
+ public Version getVersion() {
+ return Version.parseVersion(headers.get("Bundle-Version"));
+ }
+
+ private Dictionary<String, String> retrieveHeaders(URL location) throws IOException {
+ try (JarFile jarFile = new JarFile(location.getFile())) {
+ Attributes attributes = jarFile.getManifest().getMainAttributes();
+ Hashtable<String, String> ret = new Hashtable<>();
+ attributes.forEach((k, v) -> ret.put(k.toString(), v.toString()));
+ return ret;
+ }
+ }
+ }
+
+ private class BundleContextImpl implements BundleContext {
+ @Override
+ public String getProperty(String key) {
+ return null;
+ }
+
+ @Override
+ public Bundle getBundle() {
+ return systemBundleImpl;
+ }
+
+ @Override
+ public Bundle installBundle(String location, InputStream input) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Bundle installBundle(String location) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Bundle getBundle(long id) {
+ return systemBundleImpl;
+ }
+
+ @Override
+ public Bundle[] getBundles() {
+ return new Bundle[] { systemBundleImpl };
+ }
+
+ @Override
+ public Bundle getBundle(String location) {
+ return systemBundleImpl;
+ }
+
+ @Override
+ public void addServiceListener(ServiceListener listener, String filter) {
+ }
+
+ @Override
+ public void addServiceListener(ServiceListener listener) {
+ }
+
+ @Override
+ public void removeServiceListener(ServiceListener listener) {
+ }
+
+ @Override
+ public void addBundleListener(BundleListener listener) {
+ }
+
+ @Override
+ public void removeBundleListener(BundleListener listener) {
+ }
+
+ @Override
+ public void addFrameworkListener(FrameworkListener listener) {
+ }
+
+ @Override
+ public void removeFrameworkListener(FrameworkListener listener) {
+ }
+
+ @Override
+ public ServiceRegistration<?> registerService(String[] classes, Object service, Dictionary<String, ?> properties) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ServiceRegistration<?> registerService(String clazz, Object service, Dictionary<String, ?> properties) {
+ return null;
+ }
+
+ @Override
+ public <S> ServiceRegistration<S> registerService(Class<S> clazz, S service, Dictionary<String, ?> properties) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ServiceReference<?>[] getServiceReferences(String clazz, String filter) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ServiceReference<?>[] getAllServiceReferences(String clazz, String filter) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ServiceReference<?> getServiceReference(String clazz) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <S> ServiceReference<S> getServiceReference(Class<S> clazz) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <S> Collection<ServiceReference<S>> getServiceReferences(Class<S> clazz, String filter) {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public <S> S getService(ServiceReference<S> reference) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean ungetService(ServiceReference<?> reference) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public File getDataFile(String filename) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Filter createFilter(String filter) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <S> ServiceRegistration<S> registerService(Class<S> aClass, ServiceFactory<S> serviceFactory,
+ Dictionary<String, ?> dictionary) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <S> ServiceObjects<S> getServiceObjects(ServiceReference<S> serviceReference) {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
diff --git a/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java b/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java
new file mode 100644
index 00000000000..a0fee3265df
--- /dev/null
+++ b/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java
@@ -0,0 +1,34 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.application.container.impl;
+
+import com.yahoo.text.Utf8;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * @author Tony Vaagenes
+ * @author ollivir
+ */
+public class StandaloneContainerRunner {
+ public static Path createApplicationPackage(String servicesXml) {
+ try {
+ return createApplicationDirectory(servicesXml);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Path createApplicationDirectory(String servicesXml) throws IOException {
+ Path applicationDir = Files.createTempDirectory("application");
+ Path servicesXmlFile = applicationDir.resolve("services.xml");
+ String content = servicesXml;
+
+ if (!servicesXml.startsWith("<?xml")) {
+ content = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + servicesXml;
+ }
+ Files.write(servicesXmlFile, Utf8.toBytes(content));
+ return applicationDir;
+ }
+}
diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java
new file mode 100644
index 00000000000..4bbe9986d90
--- /dev/null
+++ b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java
@@ -0,0 +1,97 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.standalone;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
+import com.yahoo.net.HostName;
+
+import java.io.File;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * FileAcquirer and FileRegistry working on a local directory.
+ *
+ * @author Tony Vaagenes
+ * @author ollivir
+ */
+public class LocalFileDb implements FileAcquirer, FileRegistry {
+ private static final Constructor<FileReference> fileReferenceConstructor = createFileReferenceConstructor();
+
+ private final Map<FileReference, File> fileReferenceToFile = new HashMap<>();
+ private final Path appPath;
+
+ public LocalFileDb(Path appPath) {
+ this.appPath = appPath;
+ }
+
+ /* FileAcquirer overrides */
+ @Override
+ public File waitFor(FileReference reference, long l, TimeUnit timeUnit) {
+ synchronized (this) {
+ File file = fileReferenceToFile.get(reference);
+ if (file == null) {
+ throw new RuntimeException("Invalid file reference " + reference);
+ }
+ return file;
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ }
+
+ /* FileRegistry overrides */
+ public FileReference addFile(String relativePath) {
+ File file = appPath.resolve(relativePath).toFile();
+ if (!file.exists()) {
+ throw new RuntimeException("The file does not exist: " + file.getPath());
+ }
+
+ FileReference fileReference = null;
+ try {
+ fileReference = fileReferenceConstructor.newInstance("LocalFileDb:" + relativePath);
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
+ throw new RuntimeException("Unable to create new FileReference", e);
+ }
+ fileReferenceToFile.put(fileReference, file);
+ return fileReference;
+ }
+
+ @Override
+ public List<Entry> export() {
+ return fileReferenceToFile.entrySet().stream().map(entry -> new Entry(entry.getValue().getPath(), entry.getKey()))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public FileReference addUri(String uri) {
+ throw new RuntimeException("addUri(String uri) is not implemented here.");
+ }
+
+ public String fileSourceHost() {
+ return HostName.getLocalhost();
+ }
+
+ public Set<String> allRelativePaths() {
+ return fileReferenceToFile.values().stream().map(File::getPath).collect(Collectors.toSet());
+ }
+
+ private static Constructor<FileReference> createFileReferenceConstructor() {
+ try {
+ Constructor<FileReference> method = FileReference.class.getDeclaredConstructor(String.class);
+ method.setAccessible(true);
+ return method;
+ } catch (NoSuchMethodException ex) {
+ throw new IllegalStateException(ex);
+ }
+ }
+}
diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java
new file mode 100644
index 00000000000..72937301954
--- /dev/null
+++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java
@@ -0,0 +1,304 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.standalone;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.ConfigurationException;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.ProvisionException;
+import com.google.inject.name.Named;
+import com.google.inject.name.Names;
+import com.yahoo.collections.Pair;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.application.provider.FilesApplicationPackage;
+import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.container.di.config.SubscriberFactory;
+import com.yahoo.container.jdisc.ConfiguredApplication;
+import com.yahoo.io.IOUtils;
+import com.yahoo.jdisc.application.Application;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.container.xml.ConfigServerContainerModelBuilder;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking;
+import org.w3c.dom.Element;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static com.yahoo.collections.CollectionUtil.first;
+
+/**
+ * @author Tony Vaagenes
+ * @author gjoranv
+ * @author ollivir
+ */
+public class StandaloneContainerApplication implements Application {
+ public static final String PACKAGE_NAME = "standalone_jdisc_container";
+ public static final String APPLICATION_LOCATION_INSTALL_VARIABLE = PACKAGE_NAME + ".app_location";
+ public static final String DEPLOYMENT_PROFILE_INSTALL_VARIABLE = PACKAGE_NAME + ".deployment_profile";
+ public static final String DISABLE_NETWORKING_ANNOTATION = "JDisc.disableNetworking";
+ public static final Named APPLICATION_PATH_NAME = Names.named(APPLICATION_LOCATION_INSTALL_VARIABLE);
+ public static final Named CONFIG_MODEL_REPO_NAME = Names.named("ConfigModelRepo");
+
+ private static final String DEFAULT_TMP_BASE_DIR = Defaults.getDefaults().underVespaHome("tmp");
+ private static final String TMP_DIR_NAME = "standalone_container";
+
+ private static final StaticConfigDefinitionRepo configDefinitionRepo = new StaticConfigDefinitionRepo();
+
+ private final Injector injector;
+ private final Path applicationPath;
+ private final LocalFileDb distributedFiles;
+ private final ConfigModelRepo configModelRepo;
+ private final Networking networkingOption;
+ private final VespaModel modelRoot;
+ private final Application configuredApplication;
+ private final Container container;
+
+ @Inject
+ public StandaloneContainerApplication(Injector injector) {
+ this.injector = injector;
+ ConfiguredApplication.ensureVespaLoggingInitialized();
+ this.applicationPath = injectedApplicationPath().orElseGet(this::installApplicationPath);
+ this.distributedFiles = new LocalFileDb(applicationPath);
+ this.configModelRepo = resolveConfigModelRepo();
+ this.networkingOption = resolveNetworkingOption();
+
+ try {
+ Pair<VespaModel, Container> tpl = withTempDir(preprocessedApplicationDir -> createContainerModel(applicationPath,
+ distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo));
+ this.modelRoot = tpl.getFirst();
+ this.container = tpl.getSecond();
+ } catch (RuntimeException r) {
+ throw r;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create ContainerModel", e);
+ }
+ this.configuredApplication = createConfiguredApplication(container);
+ }
+
+ private ConfigModelRepo resolveConfigModelRepo() {
+ try {
+ return injector.getInstance(Key.get(ConfigModelRepo.class, CONFIG_MODEL_REPO_NAME));
+ } catch (Exception e) {
+ return new ConfigModelRepo();
+ }
+ }
+
+ private Networking resolveNetworkingOption() {
+ try {
+ Boolean networkingDisable = injector.getInstance(Key.get(Boolean.class, Names.named(DISABLE_NETWORKING_ANNOTATION)));
+ if (networkingDisable != null) {
+ return networkingDisable ? Networking.disable : Networking.enable;
+ }
+ } catch (Exception ignored) {
+ }
+ return Networking.enable;
+ }
+
+ private Application createConfiguredApplication(Container container) {
+ Injector augmentedInjector = injector.createChildInjector(new AbstractModule() {
+ @Override
+ public void configure() {
+ bind(SubscriberFactory.class).toInstance(new StandaloneSubscriberFactory(modelRoot));
+ }
+ });
+
+ System.setProperty("config.id", container.getConfigId());
+ return augmentedInjector.getInstance(ConfiguredApplication.class);
+ }
+
+ private Optional<Path> injectedApplicationPath() {
+ try {
+ return Optional.ofNullable(injector.getInstance(Key.get(Path.class, APPLICATION_PATH_NAME)));
+ } catch (ConfigurationException | ProvisionException ignored) {
+ }
+ return Optional.empty();
+ }
+
+ private Path installApplicationPath() {
+ Optional<String> variable = optionalInstallVariable(APPLICATION_LOCATION_INSTALL_VARIABLE);
+
+ return variable.map(Paths::get)
+ .orElseThrow(() -> new IllegalStateException("Environment variable not set: " + APPLICATION_LOCATION_INSTALL_VARIABLE));
+ }
+
+ @Override
+ public void start() {
+ try {
+ com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles);
+ configuredApplication.start();
+ } catch (Exception e) {
+ com.yahoo.container.Container.resetInstance();
+ throw e;
+ }
+ }
+
+ @Override
+ public void stop() {
+ configuredApplication.stop();
+ }
+
+ @Override
+ public void destroy() {
+ com.yahoo.container.Container.resetInstance();
+ configuredApplication.destroy();
+ }
+
+ public Container container() {
+ return container;
+ }
+
+ private interface ThrowingFunction<T, U> {
+ U apply(T input) throws Exception;
+ }
+
+ private static <T> T withTempDir(ThrowingFunction<File, T> f) throws Exception {
+ File tmpDir = createTempDir();
+ try {
+ return f.apply(tmpDir);
+ } finally {
+ IOUtils.recursiveDeleteDir(tmpDir);
+ }
+ }
+
+ private static File createTempDir() {
+ Path basePath;
+ if (new File(DEFAULT_TMP_BASE_DIR).exists()) {
+ basePath = Paths.get(DEFAULT_TMP_BASE_DIR);
+ } else {
+ basePath = Paths.get(System.getProperty("java.io.tmpdir"));
+ }
+
+ try {
+ Path tmpDir = Files.createTempDirectory(basePath, TMP_DIR_NAME);
+ return tmpDir.toFile();
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot create temp directory", e);
+ }
+ }
+
+ private static void validateApplication(ApplicationPackage applicationPackage) {
+ try {
+ applicationPackage.validateXML();
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ private static ContainerModelBuilder newContainerModelBuilder(Networking networkingOption) {
+ Optional<String> profile = optionalInstallVariable(DEPLOYMENT_PROFILE_INSTALL_VARIABLE);
+ if (profile.isPresent()) {
+ String profileName = profile.get();
+ if ("configserver".equals(profileName)) {
+ return new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables());
+ } else {
+ throw new RuntimeException("Invalid deployment profile '" + profileName + "'");
+ }
+ } else {
+ return new ContainerModelBuilder(true, networkingOption);
+ }
+ }
+
+ static Pair<VespaModel, Container> createContainerModel(Path applicationPath, FileRegistry fileRegistry,
+ File preprocessedApplicationDir, Networking networkingOption, ConfigModelRepo configModelRepo) throws Exception {
+ DeployLogger logger = new BaseDeployLogger();
+ FilesApplicationPackage rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile())
+ .includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build();
+ ApplicationPackage applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), logger);
+ validateApplication(applicationPackage);
+ DeployState deployState = new DeployState.Builder().applicationPackage(applicationPackage).fileRegistry(fileRegistry)
+ .deployLogger(logger).configDefinitionRepo(configDefinitionRepo).build(true);
+
+ VespaModel root = VespaModel.createIncomplete(deployState);
+ ApplicationConfigProducerRoot vespaRoot = new ApplicationConfigProducerRoot(root, "vespa", deployState.getDocumentModel(),
+ deployState.getProperties().vespaVersion(), deployState.getProperties().applicationId());
+
+ Element spec = containerRootElement(applicationPackage);
+ ContainerModel containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec);
+ containerModel.getCluster().prepare();
+ initializeContainerModel(containerModel, configModelRepo);
+ Container container = first(containerModel.getCluster().getContainers());
+
+ // TODO: Separate out model finalization from the VespaModel constructor,
+ // such that the above and below code to finalize the container can be
+ // replaced by root.finalize();
+
+ initializeContainer(container, spec);
+
+ root.freezeModelTopology();
+ return new Pair<>(root, container);
+ }
+
+ private static void initializeContainer(Container container, Element spec) {
+ HostResource host = container.getRoot().getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC);
+
+ container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec));
+ container.setHostResource(host);
+ container.initService();
+ }
+
+ private static Element getJDiscInServices(Element element) {
+ List<Element> jDiscElements = new ArrayList<>();
+ for (ConfigModelId cid : ContainerModelBuilder.configModelIds) {
+ List<Element> children = XML.getChildren(element, cid.getName());
+ jDiscElements.addAll(children);
+ }
+
+ if (jDiscElements.size() == 1) {
+ return jDiscElements.get(0);
+ } else if (jDiscElements.isEmpty()) {
+ throw new RuntimeException("No jdisc element found under services.");
+ } else {
+ List<String> nameAndId = jDiscElements.stream().map(e -> e.getNodeName() + " id='" + e.getAttribute("id") + "'")
+ .collect(Collectors.toList());
+ throw new RuntimeException("Found multiple JDisc elements: " + String.join(", ", nameAndId));
+ }
+ }
+
+ private static Element containerRootElement(ApplicationPackage applicationPackage) {
+ Element element = XmlHelper.getDocument(applicationPackage.getServices()).getDocumentElement();
+ String nodeName = element.getNodeName();
+
+ if (ContainerModelBuilder.configModelIds.stream().anyMatch(id -> id.getName().equals(nodeName))) {
+ return element;
+ } else {
+ return getJDiscInServices(element);
+ }
+ }
+
+ @SuppressWarnings("deprecation") // TODO: what is the not-deprecated way?
+ private static void initializeContainerModel(ContainerModel containerModel, ConfigModelRepo configModelRepo) {
+ containerModel.initialize(configModelRepo);
+ }
+
+ private static Optional<String> optionalInstallVariable(String name) {
+ Optional<String> fromEnv = Optional.ofNullable(System.getenv((name.replace(".", "__"))));
+ if (fromEnv.isPresent()) {
+ return fromEnv;
+ }
+ return Optional.ofNullable(System.getProperty(name)); // for unit testing
+ }
+}
diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java
new file mode 100644
index 00000000000..6ea2671b05b
--- /dev/null
+++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java
@@ -0,0 +1,131 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.standalone;
+
+import com.yahoo.config.ConfigBuilder;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.subscription.ConfigInterruptedException;
+import com.yahoo.container.di.config.Subscriber;
+import com.yahoo.container.di.config.SubscriberFactory;
+import com.yahoo.vespa.config.ConfigKey;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author Tony Vaagenes
+ * @author gjoranv
+ * @author ollivir
+ */
+public class StandaloneSubscriberFactory implements SubscriberFactory {
+ private final VespaModel root;
+
+ public StandaloneSubscriberFactory(VespaModel root) {
+ this.root = root;
+ }
+
+ private class StandaloneSubscriber implements Subscriber {
+
+ private final Set<ConfigKey<ConfigInstance>> configKeys;
+ private long generation = -1L;
+
+ StandaloneSubscriber(Set<ConfigKey<ConfigInstance>> configKeys) {
+ this.configKeys = configKeys;
+ }
+
+ @Override
+ public boolean internalRedeploy() { return false; }
+
+ @Override
+ public boolean configChanged() {
+ return generation == 0;
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public Map<ConfigKey<ConfigInstance>, ConfigInstance> config() {
+ Map<ConfigKey<ConfigInstance>, ConfigInstance> ret = new HashMap<>();
+ for (ConfigKey<ConfigInstance> key : configKeys) {
+ ConfigInstance.Builder builder = root.getConfig(newBuilderInstance(key), key.getConfigId());
+ if (builder == null) {
+ throw new RuntimeException("Invalid config id " + key.getConfigId());
+ }
+ ret.put(key, newConfigInstance(builder));
+ }
+ return ret;
+ }
+
+ @Override
+ public long waitNextGeneration() {
+ generation++;
+
+ if (generation != 0) {
+ try {
+ while (!Thread.interrupted()) {
+ Thread.sleep(10000);
+ }
+ } catch (InterruptedException e) {
+ throw new ConfigInterruptedException(e);
+ }
+ }
+
+ return generation;
+ }
+
+ // if waitNextGeneration has not yet been called, -1 should be returned
+ @Override
+ public long generation() {
+ return generation;
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Subscriber getSubscriber(Set<? extends ConfigKey<?>> configKeys) {
+ return new StandaloneSubscriber((Set<ConfigKey<ConfigInstance>>) configKeys);
+ }
+
+ public void reloadActiveSubscribers(long generation) {
+ throw new RuntimeException("unsupported");
+ }
+
+ private static ConfigInstance.Builder newBuilderInstance(ConfigKey<ConfigInstance> key) {
+ try {
+ return builderClass(key).getDeclaredConstructor().newInstance();
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ throw new RuntimeException("ConfigInstance builder cannot be instantiated", e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Class<ConfigInstance.Builder> builderClass(ConfigKey<ConfigInstance> key) {
+ Class<?> configClass = key.getConfigClass();
+ if (configClass != null) {
+ Class<?>[] nestedClasses = configClass.getClasses();
+ for (Class<?> clazz : nestedClasses) {
+ if (clazz.getName().equals(key.getConfigClass().getName() + "$Builder")) {
+ return (Class<ConfigInstance.Builder>) clazz;
+ }
+ }
+ }
+ throw new RuntimeException("Builder class for " + (configClass == null ? null : configClass.getName()) + " could not be located");
+ }
+
+ private static ConfigInstance newConfigInstance(ConfigBuilder builder) {
+ try {
+ return configClass(builder).getConstructor(builder.getClass()).newInstance(builder);
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ throw new RuntimeException("ConfigInstance cannot be instantiated", e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Class<ConfigInstance> configClass(ConfigBuilder builder) {
+ return (Class<ConfigInstance>) builder.getClass().getEnclosingClass();
+ }
+}
diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala
deleted file mode 100644
index ac8636de2cb..00000000000
--- a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala
+++ /dev/null
@@ -1,206 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.application.container.impl
-
-import java.io.InputStream
-import java.net.{URL, URLClassLoader}
-import java.util
-import java.util.concurrent.atomic.AtomicInteger
-import java.util.jar.JarFile
-import java.util.{Collections, Dictionary, Hashtable}
-
-import com.yahoo.container.standalone.StandaloneContainerApplication
-import com.yahoo.jdisc.application.{OsgiFramework, OsgiHeader}
-import org.osgi.framework._
-import org.osgi.framework.wiring._
-import org.osgi.resource.{Capability, Requirement, Wire}
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-/**
- * A (mock) OSGI implementation which loads classes from the system classpath
- *
- * @author tonytv
- */
-final class ClassLoaderOsgiFramework extends OsgiFramework {
- private val bundleLocations = new ArrayBuffer[URL]
- private val bundleList = ArrayBuffer[Bundle](SystemBundleImpl)
- private var classLoader: ClassLoader = null
-
- private val nextBundleId = new AtomicInteger(1)
-
- override def installBundle(bundleLocation: String) = {
- if (bundleLocation != "") {
- val url = new URL(bundleLocation)
- bundleLocations += url
- bundleList += new JarBundleImpl(url)
- }
-
- bundles()
- }
-
- def getClassLoader = {
- if (bundleLocations.isEmpty) {
- getClass.getClassLoader
- } else {
- if(classLoader == null)
- classLoader = new URLClassLoader(bundleLocations.toArray, getClass.getClassLoader)
-
- classLoader
- }
- }
-
- override def startBundles(bundles: util.List[Bundle], privileged: Boolean) {}
-
- override def refreshPackages() {}
-
- override def bundleContext():BundleContext = BundleContextImpl
-
- override def bundles() = bundleList.asJava
-
- override def start() {}
-
- override def stop() {}
-
- private abstract class BundleImpl extends Bundle {
- override def getState = Bundle.ACTIVE
-
- override def start(options: Int) {}
- override def start() {}
- override def stop(options: Int) {}
- override def stop() {}
- override def update(input: InputStream) {}
- override def update() {}
- override def uninstall() {}
-
- override def getHeaders(locale: String) = getHeaders
-
- override def getSymbolicName = ClassLoaderOsgiFramework.this.getClass.getName
- override def getLocation = getSymbolicName
-
- override def getRegisteredServices = Array[ServiceReference[_]]()
- override def getServicesInUse = getRegisteredServices
-
- override def hasPermission(permission: Any) = true
-
- override def getResource(name: String) = getClassLoader.getResource(name)
- override def loadClass(name: String) = getClassLoader.loadClass(name)
- override def getResources(name: String) = getClassLoader.getResources(name)
-
- override def getEntryPaths(path: String) = throw new UnsupportedOperationException
- override def getEntry(path: String) = throw new UnsupportedOperationException
- override def findEntries(path: String, filePattern: String, recurse: Boolean) = throw new UnsupportedOperationException
-
- override def getLastModified = 1L
-
- override def getBundleContext = throw new UnsupportedOperationException
- override def getSignerCertificates(signersType: Int) = Collections.emptyMap()
-
- override def adapt[A](`type`: Class[A]): A = {
- if (`type` == classOf[BundleRevision]) BundleRevisionImpl.asInstanceOf[A]
- else if (`type` == classOf[BundleWiring]) BundleWiringImpl.asInstanceOf[A]
- else null.asInstanceOf[A]
- }
-
- override def getDataFile(filename: String) = null
- override def compareTo(o: Bundle) = getBundleId compareTo o.getBundleId
- }
-
- private object BundleRevisionImpl extends BundleRevision {
- override def getSymbolicName: String = this.getClass.getName
- override def getDeclaredRequirements(p1: String): util.List[BundleRequirement] = throw new UnsupportedOperationException
- override def getVersion: Version = Version.emptyVersion
- override def getWiring: BundleWiring = BundleWiringImpl
- override def getDeclaredCapabilities(p1: String): util.List[BundleCapability] = throw new UnsupportedOperationException
- override def getTypes: Int = 0
- override def getBundle: Bundle = throw new UnsupportedOperationException
- override def getCapabilities(p1: String): util.List[Capability] = throw new UnsupportedOperationException
- override def getRequirements(p1: String): util.List[Requirement] = throw new UnsupportedOperationException
- }
-
- private object BundleWiringImpl extends BundleWiring {
- override def findEntries(p1: String, p2: String, p3: Int): util.List[URL] = ???
- override def getRequiredResourceWires(p1: String): util.List[Wire] = ???
- override def getResourceCapabilities(p1: String): util.List[Capability] = ???
- override def isCurrent: Boolean = ???
- override def getRequiredWires(p1: String): util.List[BundleWire] = ???
- override def getCapabilities(p1: String): util.List[BundleCapability] = ???
- override def getProvidedResourceWires(p1: String): util.List[Wire] = ???
- override def getProvidedWires(p1: String): util.List[BundleWire] = ???
- override def getRevision: BundleRevision = ???
- override def getResourceRequirements(p1: String): util.List[Requirement] = ???
- override def isInUse: Boolean = ???
- override def listResources(p1: String, p2: String, p3: Int): util.Collection[String] = ???
- override def getClassLoader: ClassLoader = ClassLoaderOsgiFramework.this.getClassLoader
- override def getRequirements(p1: String): util.List[BundleRequirement] = ???
- override def getResource: BundleRevision = ???
- override def getBundle: Bundle = ???
- }
-
- private object SystemBundleImpl extends BundleImpl {
- override val getBundleId = 0L
- override def getVersion = Version.emptyVersion
- override def getHeaders: Dictionary[String, String] = new Hashtable[String, String](Map(OsgiHeader.APPLICATION -> classOf[StandaloneContainerApplication].getName).asJava)
- }
-
-
- private class JarBundleImpl(location: URL) extends BundleImpl {
- override val getBundleId = nextBundleId.getAndIncrement.asInstanceOf[Long]
-
- private val headers = retrieveHeaders(location)
-
- override def getHeaders: Dictionary[String, String] = headers
- override val getSymbolicName = headers.get("Bundle-SymbolicName")
- override val getVersion = Version.parseVersion(headers.get("Bundle-Version"))
-
-
- private def retrieveHeaders(location: URL) = {
- val jarFile = new JarFile(location.getFile)
- try {
- val attributes = jarFile.getManifest.getMainAttributes
- new Hashtable[String, String](attributes.entrySet().asScala.map( entry => entry.getKey.toString -> entry.getValue.toString).toMap.asJava)
- } finally {
- jarFile.close()
- }
- }
- }
-
- private object BundleContextImpl extends BundleContext {
- private val bundleImpl = SystemBundleImpl
-
- override def getProperty(key: String) = null
- override def getBundle = bundleImpl
- override def installBundle(location: String, input: InputStream) = throw new UnsupportedOperationException
- override def installBundle(location: String) = throw new UnsupportedOperationException
-
- override def getBundle(id: Long) = bundleImpl
- override def getBundles = Array(bundleImpl)
- override def getBundle(location: String) = bundleImpl
-
- override def addServiceListener(listener: ServiceListener, filter: String) {}
- override def addServiceListener(listener: ServiceListener) {}
- override def removeServiceListener(listener: ServiceListener) {}
- override def addBundleListener(listener: BundleListener) {}
- override def removeBundleListener(listener: BundleListener) {}
-
- override def addFrameworkListener(listener: FrameworkListener) {}
- override def removeFrameworkListener(listener: FrameworkListener) {}
-
- override def registerService(clazzes: Array[String], service: Any, properties: Dictionary[String, _]) = throw new UnsupportedOperationException
- override def registerService(clazz: String, service: Any, properties: Dictionary[String, _]) = null
- override def registerService[S](clazz: Class[S], service: S, properties: Dictionary[String, _]) = throw new UnsupportedOperationException
- override def getServiceReferences(clazz: String, filter: String) = throw new UnsupportedOperationException
- override def getAllServiceReferences(clazz: String, filter: String) = throw new UnsupportedOperationException
- override def getServiceReference(clazz: String) = throw new UnsupportedOperationException
- override def getServiceReference[S](clazz: Class[S]) = throw new UnsupportedOperationException
- override def getServiceReferences[S](clazz: Class[S], filter: String) = Collections.emptyList()
- override def getService[S](reference: ServiceReference[S]) = throw new UnsupportedOperationException
- override def ungetService(reference: ServiceReference[_]) = throw new UnsupportedOperationException
- override def getDataFile(filename: String) = throw new UnsupportedOperationException
- override def createFilter(filter: String) = throw new UnsupportedOperationException
-
- override def registerService[S](aClass: Class[S], serviceFactory: ServiceFactory[S], dictionary: Dictionary[String, _]): ServiceRegistration[S] = throw new UnsupportedOperationException
- override def getServiceObjects[S](serviceReference: ServiceReference[S]): ServiceObjects[S] = throw new UnsupportedOperationException
- }
-
-}
diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala
deleted file mode 100644
index 91634250fc5..00000000000
--- a/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.application.container.impl
-
-import java.nio.file.Files
-import com.yahoo.text.Utf8
-
-/**
- * @author tonytv
- */
-final class StandaloneContainerRunner {
-
-
-
-}
-
-object StandaloneContainerRunner {
- def createApplicationPackage(servicesXml: String) = {
- val applicationDir = Files.createTempDirectory("application")
-
- val servicesXmlFile = applicationDir.resolve("services.xml");
- var content = servicesXml;
- if ( ! servicesXml.startsWith("<?xml"))
- content = """<?xml version="1.0" encoding="utf-8" ?>""" + '\n' + servicesXml
- Files.write(servicesXmlFile, Utf8.toBytes(content))
- applicationDir
- }
-}
diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala
deleted file mode 100644
index 61128347319..00000000000
--- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-/**
- * @author tonytv
- */
-trait Converter[T] {
- def convert(s: String): T
-}
-
-object Converter {
- def toConverter[T](f: String => T) = new Converter[T] {
- override def convert(s: String) = f(s)
- }
-
- implicit val intConverter = toConverter(_.toInt)
- implicit val longConverter = toConverter(_.toLong)
- implicit val boolConverter = toConverter(_.toBoolean)
- implicit val stringConverter = toConverter(identity)
-
- implicit val javaIntegerConverter:Converter[Integer] = toConverter(_.toInt)
- implicit val javaLongConverter:Converter[java.lang.Long] = toConverter(_.toLong)
- implicit val javaBooleanConverter:Converter[java.lang.Boolean] = toConverter(_.toBoolean)
-
-
-}
diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala
deleted file mode 100644
index 2aab88d8319..00000000000
--- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-/**
- * @author tonytv
- * TODO: copied from standalone-container. Move to separate lib module instead.
- */
-object Environment {
- def optionalInstallVariable(name: String) = {
- env(name.replace(".", "__")).
- orElse(systemProperty(name)) //for unit testing
- }
-
- def installVariable(name: String) = {
- optionalInstallVariable(name).
- getOrElse {
- throw new IllegalStateException("Environment variable not set: " + name)
- }
- }
-
- def env(name: String) = Option(System.getenv(name))
- def systemProperty(name: String) = Option(System.getProperty(name))
-}
diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala
deleted file mode 100644
index 6507b4c72f0..00000000000
--- a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-import java.io.File
-import java.lang.reflect.Constructor
-import java.nio.file.Path
-import java.util
-import java.util.concurrent.TimeUnit
-
-import com.yahoo.config.FileReference
-import com.yahoo.config.application.api.FileRegistry
-import com.yahoo.config.application.api.FileRegistry.Entry
-import com.yahoo.container.standalone.LocalFileDb._
-import com.yahoo.filedistribution.fileacquirer.FileAcquirer
-import com.yahoo.net.HostName
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-
-/**
- * FileAcquirer and FileRegistry working on a local directory.
- * @author tonytv
- */
-class LocalFileDb(appPath: Path) extends FileAcquirer with FileRegistry {
- private val fileReferenceToFile = mutable.Map[FileReference, File]()
-
- /** *** FileAcquirer overrides *****/
- def waitFor(reference: FileReference, l: Long, timeUnit: TimeUnit): File = {
- synchronized {
- fileReferenceToFile.get(reference).getOrElse {
- throw new RuntimeException("Invalid file reference " + reference)
- }
- }
- }
-
- override def shutdown() {}
-
- /** *** FileRegistry overrides *****/
- def addFile(relativePath: String): FileReference = {
- val file = appPath.resolve(relativePath).toFile
- if (!file.exists) {
- throw new RuntimeException("The file does not exist: " + file.getPath)
- }
-
- val fileReference: FileReference = fileReferenceConstructor.newInstance("LocalFileDb:" + relativePath)
- fileReferenceToFile.put(fileReference, file)
- fileReference
- }
-
- def fileSourceHost: String =
- HostName.getLocalhost
-
- def allRelativePaths: java.util.Set[String] = {
- new java.util.HashSet(fileReferenceToFile.values.map(_.getPath).asJavaCollection)
- }
-
- override def export(): util.List[Entry] = {
- new java.util.ArrayList(fileReferenceToFile.keys.map{ (ref: FileReference) => new Entry(fileReferenceToFile.get(ref).get.getPath, ref)}.asJavaCollection)
- }
-
- override def addUri(uri: String): FileReference = {
- throw new RuntimeException("addUri(uri: String) is not implemented here.");
- }
-}
-
-object LocalFileDb {
- private def createFileReferenceConstructor: Constructor[FileReference] = {
- val method: Constructor[FileReference] = classOf[FileReference].getDeclaredConstructor(classOf[String])
- method.setAccessible(true)
- method
- }
-
- private val fileReferenceConstructor: Constructor[FileReference] = createFileReferenceConstructor
-}
diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala
deleted file mode 100644
index 5271cd400d4..00000000000
--- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala
+++ /dev/null
@@ -1,231 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-import java.io.{File, IOException}
-import java.lang.{Boolean => JBoolean}
-import java.nio.file.{FileSystems, Files, Path, Paths}
-
-import com.google.inject.name.Names
-import com.google.inject.{AbstractModule, Inject, Injector, Key}
-import com.yahoo.collections.CollectionUtil.first
-import com.yahoo.config.application.api.{ApplicationPackage, FileRegistry}
-import com.yahoo.config.model.application.provider._
-import com.yahoo.config.model.builder.xml.XmlHelper
-import com.yahoo.config.model.deploy.DeployState
-import com.yahoo.config.model.{ApplicationConfigProducerRoot, ConfigModelRepo}
-import com.yahoo.config.provision.Zone
-import com.yahoo.container.di.config.SubscriberFactory
-import com.yahoo.container.jdisc.ConfiguredApplication
-import com.yahoo.container.standalone.Environment._
-import com.yahoo.container.standalone.StandaloneContainerApplication._
-import com.yahoo.io.IOUtils
-import com.yahoo.jdisc.application.Application
-import com.yahoo.text.XML
-import com.yahoo.vespa.defaults.Defaults
-import com.yahoo.vespa.model.VespaModel
-import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder
-import com.yahoo.vespa.model.container.{Container, ContainerModel}
-import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking
-import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ContainerModelBuilder}
-import org.w3c.dom.Element
-
-import scala.collection.JavaConverters._
-import scala.util.Try
-
-/**
- * @author Tony Vaagenes
- * @author gjoranv
- */
-class StandaloneContainerApplication @Inject()(injector: Injector) extends Application {
-
- ConfiguredApplication.ensureVespaLoggingInitialized()
-
- val applicationPath: Path = injectedApplicationPath.getOrElse(installApplicationPath)
-
- val distributedFiles = new LocalFileDb(applicationPath)
-
- val configModelRepo = Try { injector.getInstance(Key.get(classOf[ConfigModelRepo], configModelRepoName))}.getOrElse(new ConfigModelRepo)
-
- val networkingOption = Try {
- injector.getInstance(Key.get(classOf[JBoolean], Names.named(disableNetworkingAnnotation)))
- }.map {
- case JBoolean.TRUE => Networking.disable
- case JBoolean.FALSE => Networking.enable
- }.getOrElse(Networking.enable)
-
- val (modelRoot, container) = withTempDir(
- preprocessedApplicationDir => createContainerModel(applicationPath, distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo))
-
- val configuredApplication = createConfiguredApplication(container)
-
- def createConfiguredApplication(container: Container): Application = {
- val augmentedInjector = injector.createChildInjector(new AbstractModule {
- def configure() {
- bind(classOf[SubscriberFactory]).toInstance(new StandaloneSubscriberFactory(modelRoot))
- }
- })
-
- System.setProperty("config.id", container.getConfigId) //TODO: DRY
- augmentedInjector.getInstance(classOf[ConfiguredApplication])
- }
-
- def injectedApplicationPath = Try {
- injector.getInstance(Key.get(classOf[Path], applicationPathName))
- }.toOption
-
- def installApplicationPath = path(installVariable(applicationLocationInstallVariable))
-
- override def start() {
- try {
- com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles)
- configuredApplication.start()
- }
- catch {
- case e: Exception => { com.yahoo.container.Container.resetInstance(); throw e; }
- }
- }
-
- override def stop() {
- configuredApplication.stop()
- }
-
- override def destroy() {
- com.yahoo.container.Container.resetInstance()
- configuredApplication.destroy()
- }
-}
-
-object StandaloneContainerApplication {
- val packageName = "standalone_jdisc_container"
- val applicationLocationInstallVariable = s"$packageName.app_location"
- val deploymentProfileInstallVariable = s"$packageName.deployment_profile"
-
- val applicationPathName = Names.named(applicationLocationInstallVariable)
-
- val disableNetworkingAnnotation = "JDisc.disableNetworking"
- val configModelRepoName = Names.named("ConfigModelRepo")
- val configDefinitionRepo = new StaticConfigDefinitionRepo()
-
- val defaultTmpBaseDir = Defaults.getDefaults().underVespaHome("tmp")
- val tmpDirName = "standalone_container"
-
- private def withTempDir[T](f: File => T): T = {
- val tmpDir = createTempDir()
- try {
- f(tmpDir)
- } finally {
- IOUtils.recursiveDeleteDir(tmpDir)
- }
- }
-
- private def createTempDir(): File = {
- def getBaseDir: Path = {
- val tmpBaseDir =
- if (new File(defaultTmpBaseDir).exists())
- defaultTmpBaseDir
- else
- System.getProperty("java.io.tmpdir")
-
- Paths.get(tmpBaseDir)
- }
-
- val basePath: Path = getBaseDir
- val tmpDir = Files.createTempDirectory(basePath, tmpDirName)
- tmpDir.toFile
- }
-
- private def validateApplication(applicationPackage: ApplicationPackage) = {
- try {
- applicationPackage.validateXML()
- } catch {
- case e: IOException => throw new IllegalArgumentException(e)
- }
- }
-
- def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = {
- optionalInstallVariable(deploymentProfileInstallVariable) match {
- case None => new ContainerModelBuilder(true, networkingOption)
- case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables)
- case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'")
- }
- }
-
- def createContainerModel(applicationPath: Path,
- fileRegistry: FileRegistry,
- preprocessedApplicationDir: File,
- networkingOption: Networking,
- configModelRepo: ConfigModelRepo = new ConfigModelRepo): (VespaModel, Container) = {
- val logger = new BaseDeployLogger
- val rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile).includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build()
- val applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), logger)
- validateApplication(applicationPackage)
- val deployState = new DeployState.Builder().
- applicationPackage(applicationPackage).
- fileRegistry(fileRegistry).
- deployLogger(logger).
- configDefinitionRepo(configDefinitionRepo).
- build(true)
-
- val root = VespaModel.createIncomplete(deployState)
- val vespaRoot = new ApplicationConfigProducerRoot(root,
- "vespa",
- deployState.getDocumentModel,
- deployState.getProperties.vespaVersion(),
- deployState.getProperties.applicationId())
-
- val spec = containerRootElement(applicationPackage)
- val containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec)
- containerModel.getCluster().prepare()
- DeprecationSuppressor.initializeContainerModel(containerModel, configModelRepo)
- val container = first(containerModel.getCluster().getContainers)
-
- // TODO: Separate out model finalization from the VespaModel constructor,
- // such that the above and below code to finalize the container can be
- // replaced by root.finalize();
-
- initializeContainer(container, spec)
-
- root.freezeModelTopology()
- (root, container)
- }
-
- def initializeContainer(container: Container, spec: Element) {
- val host = container.getRoot.getHostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)
-
- container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec))
- container.setHostResource(host)
- container.initService()
- }
-
- def getJDiscInServices(element: Element): Element = {
- def nameAndId(elements: List[Element]): List[String] = {
- elements map { e => s"${e.getNodeName} id='${e.getAttribute("id")}'" }
- }
-
- val jDiscElements = ContainerModelBuilder.configModelIds.asScala flatMap { name => XML.getChildren(element, name.getName).asScala }
- jDiscElements.toList match {
- case List(e) => e
- case Nil => throw new RuntimeException("No jdisc element found under services.")
- case multipleElements: List[Element] => throw new RuntimeException("Found multiple JDisc elements: " + nameAndId(multipleElements).mkString(", "))
- }
- }
-
- def containerRootElement(applicationPackage: ApplicationPackage) : Element = {
- val element = XmlHelper.getDocument(applicationPackage.getServices).getDocumentElement
- val nodeName = element.getNodeName
-
- if (ContainerModelBuilder.configModelIds.asScala.map(_.getName).contains(nodeName)) element
- else getJDiscInServices(element)
- }
-
- def path(s: String) = FileSystems.getDefault.getPath(s)
-
- // Ugly hack required since Scala cannot suppress warnings. https://issues.scala-lang.org/browse/SI-7934
- private object DeprecationSuppressor extends DeprecationSuppressor
- @deprecated("", "")
- private class DeprecationSuppressor {
- def initializeContainerModel(containerModel: ContainerModel, configModelRepo: ConfigModelRepo): Unit = {
- containerModel.initialize(configModelRepo)
- }
- }
-}
diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala
deleted file mode 100644
index 1bc95f52313..00000000000
--- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-import com.yahoo.config.{ConfigBuilder, ConfigInstance}
-import com.yahoo.container.di.ConfigKeyT
-import com.yahoo.container.di.config.{Subscriber, SubscriberFactory}
-import com.yahoo.container.standalone.StandaloneSubscriberFactory._
-import com.yahoo.vespa.config.ConfigKey
-import com.yahoo.vespa.model.VespaModel
-
-import scala.collection.JavaConverters._
-
-/**
- * @author tonytv
- * @author gjoranv
- */
-class StandaloneSubscriberFactory(root: VespaModel) extends SubscriberFactory {
- class StandaloneSubscriber(configKeys: Set[ConfigKeyT]) extends Subscriber {
- override def configChanged =
- generation == 0
-
- override def close() {}
-
- override def internalRedeploy() = { false }
-
- override def config = {
-
- def getConfig(key: ConfigKeyT) = {
- val builderWithModelConfig = root.getConfig(newBuilderInstance(key), key.getConfigId)
-
- require(builderWithModelConfig != null, "Invalid config id " + key.getConfigId )
- (key.asInstanceOf[ConfigKey[ConfigInstance]], newConfigInstance(builderWithModelConfig))
- }
-
- (configKeys map getConfig).toMap.asJava
- }
-
- override def waitNextGeneration() = {
- generation += 1
-
- if (generation != 0) {
- while (!Thread.interrupted())
- Thread.sleep(10000)
- }
-
- generation
- }
-
- //if waitNextGeneration has not yet been called, -1 should be returned
- var generation = -1L
-
- }
-
- override def getSubscriber(configKeys: java.util.Set[_ <: ConfigKey[_]]) =
- new StandaloneSubscriber(configKeys.asScala.toSet.asInstanceOf[Set[ConfigKeyT]])
-
- def reloadActiveSubscribers(generation: Long) {
- throw new RuntimeException("unsupported")
- }
-}
-
-object StandaloneSubscriberFactory {
-
- private def newBuilderInstance(key: ConfigKeyT) =
- builderClass(key).getDeclaredConstructor().newInstance()
-
- private def builderClass(key: ConfigKeyT) = {
- val nestedClasses = key.getConfigClass.getClasses
- nestedClasses.
- filter {_.getName.equals(key.getConfigClass.getName + "$Builder")}.
- head.
- asInstanceOf[Class[ConfigInstance.Builder]]
- }
-
- private def newConfigInstance(builder: ConfigBuilder) =
- configClass(builder).getConstructor(builder.getClass).newInstance(builder)
-
- private def configClass(builder: ConfigBuilder) =
- builder.getClass.getEnclosingClass.asInstanceOf[Class[ConfigInstance]]
-
-}
diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java
new file mode 100644
index 00000000000..1cd110d8106
--- /dev/null
+++ b/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java
@@ -0,0 +1,67 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.standalone;
+
+import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigModelsPluginDir;
+import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigServer;
+import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigServers;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Ulf Lilleengen
+ * @author Tony Vaagenes
+ */
+public class CloudConfigInstallVariablesTest {
+
+ @Test
+ public void test_configserver_parsing() {
+ CloudConfigOptions.ConfigServer[] parsed = toConfigServers("myhost.mydomain.com");
+ assertThat(parsed.length, is(1));
+ }
+
+ @Test
+ public void port_can_be_configured() {
+ CloudConfigOptions.ConfigServer[] parsed = toConfigServers("myhost:123");
+ int port = parsed[0].port.get();
+ assertThat(port, is(123));
+ }
+
+ @Test
+ public void multiple_spaces_are_supported() {
+ CloudConfigOptions.ConfigServer[] parsed = toConfigServers("test1 test2");
+ assertThat(parsed.length, is(2));
+
+ List<String> hostNames = Arrays.stream(parsed).map(cs -> cs.hostName).collect(Collectors.toList());
+ assertThat(hostNames, contains("test1", "test2"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void missing_port_gives_exception() {
+ toConfigServer("myhost:");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void non_numeric_port_gives_exception() {
+ toConfigServer("myhost:non-numeric");
+ }
+
+ @Test
+ public void string_arrays_are_split_on_spaces() {
+ String[] parsed = toConfigModelsPluginDir("/home/vespa/foo /home/vespa/bar ");
+ assertThat(parsed.length, is(2));
+ }
+
+ @Test
+ public void string_arrays_are_split_on_comma() {
+ String[] parsed = toConfigModelsPluginDir("/home/vespa/foo,/home/vespa/bar,");
+ assertThat(parsed.length, is(2));
+ }
+}
diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java
new file mode 100644
index 00000000000..a00ffd8b985
--- /dev/null
+++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java
@@ -0,0 +1,61 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.standalone;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.io.IOUtils;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Creates a local application from vespa-services fragments.
+ *
+ * @author Tony Vaagenes
+ * @author ollivir
+ */
+public class StandaloneContainer {
+ public static String firstContainerId(AbstractConfigProducerRoot root) {
+ return root.getConfigProducer("container").get().getConfigId();
+ }
+
+ interface ThrowingFunction<T, U> {
+ U apply(T input) throws Exception;
+ }
+
+ static <T> T withContainerModel(String servicesXml, ThrowingFunction<VespaModel, T> f) throws Exception {
+ return withTempDirectory(applicationPath -> {
+ writeServicesXml(applicationPath, servicesXml);
+
+ LocalFileDb distributedFiles = new LocalFileDb(applicationPath);
+ VespaModel root;
+ Pair<VespaModel, com.yahoo.vespa.model.container.Container> rc = StandaloneContainerApplication.createContainerModel(
+ applicationPath, distributedFiles, applicationPath.resolve("preprocesedApp").toFile(), Networking.enable,
+ new ConfigModelRepo());
+ root = rc.getFirst();
+ return f.apply(root);
+ });
+ }
+
+ private static <T> T withTempDirectory(ThrowingFunction<Path, T> f) throws Exception {
+ Path directory = Files.createTempDirectory("application");
+ try {
+ return f.apply(directory);
+ } finally {
+ IOUtils.recursiveDeleteDir(directory.toFile());
+ }
+ }
+
+ private static void writeServicesXml(Path applicationPath, String servicesXml) throws IOException {
+ Path path = applicationPath.resolve("services.xml");
+ List<String> output = Arrays.asList("<?xml version=\"1.0\" encoding=\"utf-8\"?>", servicesXml);
+ Files.write(path, output, StandardCharsets.UTF_8);
+ }
+}
diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java
index 8d413ade0f0..71668b595a0 100644
--- a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java
+++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.container.standalone;
import com.google.inject.Module;
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertThat;
/**
* @author Einar M R Rosenvinge
- * @since 5.22.0
*/
public class StandaloneContainerActivatorTest {
@@ -100,7 +99,7 @@ public class StandaloneContainerActivatorTest {
private static Module newAppDirBinding(final Path applicationDir) {
return binder -> binder.bind(Path.class)
- .annotatedWith(StandaloneContainerApplication.applicationPathName())
+ .annotatedWith(StandaloneContainerApplication.APPLICATION_PATH_NAME)
.toInstance(applicationDir);
}
diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java
new file mode 100644
index 00000000000..6d4abc84dbc
--- /dev/null
+++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java
@@ -0,0 +1,74 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.standalone;
+
+import com.yahoo.vespa.model.AbstractService;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Tony Vaagenes
+ * @author gjoranv
+ * @author ollivir
+ */
+
+public class StandaloneContainerTest {
+ private static final String PLAIN_XML = "<container version=\"1.0\" />";
+
+ @Test
+ public void container_is_allowed_root_element() throws Exception {
+ StandaloneContainer.withContainerModel(PLAIN_XML, root -> null);
+ }
+
+ @Test
+ public void services_is_allowed_root_element() throws Exception {
+ String servicesXml = "<services>" + //
+ "<container version=\"1.0\" />" + //
+ "</services>";
+
+ StandaloneContainer.withContainerModel(servicesXml, root -> null);
+ }
+
+ @Test(expected = Exception.class)
+ public void multiple_container_elements_cannot_be_deployed() throws Exception {
+ String twoContainersXml = "<services>" + //
+ "<container id=\"container-1\" version=\"1.0\" />" + //
+ "<container id=\"container-2\" version=\"1.0\" />" + //
+ "</services>";
+
+ StandaloneContainer.withContainerModel(twoContainersXml, root -> null);
+ }
+
+ @Test
+ public void application_preprocessor_is_run() throws Exception {
+ String servicesXml = "<services xmlns:preprocess=\"properties\">" + //
+ "<preprocess:properties>" + //
+ "<container_id>container-1</container_id>" + //
+ "</preprocess:properties>" + //
+ "<container id=\"${container_id}\" version=\"1.0\" />" + //
+ "</services>";
+
+ StandaloneContainer.withContainerModel(servicesXml, root -> {
+ assertTrue(root.getConfigProducer("container-1/standalone").isPresent());
+ return null;
+ });
+ }
+
+ @Test
+ public void no_default_ports_are_enabled_when_using_http() throws Exception {
+ String xml = "<jdisc version=\"1.0\">" + //
+ "<http>" + //
+ "<server port=\"4000\" id=\"server1\" />" + //
+ "</http>" + //
+ "</jdisc>";
+
+ StandaloneContainer.withContainerModel(xml, root -> {
+ AbstractService container = (AbstractService) root.getConfigProducer("jdisc/standalone").get();
+ System.out.println("portCnt: " + container.getPortCount());
+ System.out.println("numPorts: " + container.getNumPortsAllocated());
+ assertEquals(1, container.getNumPortsAllocated());
+ return null;
+ });
+ }
+}
diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java
new file mode 100644
index 00000000000..dd755e8e6dd
--- /dev/null
+++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java
@@ -0,0 +1,52 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.standalone;
+
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.container.BundlesConfig;
+import com.yahoo.container.ComponentsConfig;
+import com.yahoo.container.di.config.Subscriber;
+import com.yahoo.vespa.config.ConfigKey;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static com.yahoo.container.standalone.StandaloneContainer.withContainerModel;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.number.OrderingComparison.greaterThan;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Tony Vaagenes
+ * @author ollivir
+ */
+public class StandaloneSubscriberTest {
+ private static ConfigKey<ConfigInstance> bundlesKey = key("bundles");
+ private static ConfigKey<ConfigInstance> componentsKey = key("components");
+
+ private static ConfigKey<ConfigInstance> key(String name) {
+ return new ConfigKey<>(name, "container", "container");
+ }
+
+ @Test
+ @Ignore
+ public void standalone_subscriber() throws Exception {
+ withContainerModel("<container version=\"1.0\"></container>", root -> {
+ Set<ConfigKey<ConfigInstance>> keys = new HashSet<>();
+ keys.add(bundlesKey);
+ keys.add(componentsKey);
+ Subscriber subscriber = new StandaloneSubscriberFactory(root).getSubscriber(keys);
+ Map<ConfigKey<ConfigInstance>, ConfigInstance> config = subscriber.config();
+ assertThat(config.size(), is(2));
+
+ BundlesConfig bundlesConfig = (BundlesConfig) config.get(bundlesKey);
+ ComponentsConfig componentsConfig = (ComponentsConfig) config.get(componentsKey);
+
+ assertThat(bundlesConfig.bundle().size(), is(0));
+ assertThat(componentsConfig.components().size(), greaterThan(10));
+ return null;
+ });
+ }
+}
diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala
deleted file mode 100644
index d4baea43ba2..00000000000
--- a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-import com.yahoo.container.standalone.CloudConfigInstallVariables.{toConfigModelsPluginDir, toConfigServer, toConfigServers}
-import org.hamcrest.CoreMatchers.is
-import org.hamcrest.Matchers.arrayContaining
-import org.junit.Assert.assertThat
-import org.junit.Test
-
-/**
- * @author Ulf Lilleengen
- * @author Tony Vaagenes
- */
-class CloudConfigInstallVariablesTest {
-
- @Test
- def test_configserver_parsing {
- val parsed = toConfigServers("myhost.mydomain.com")
- assertThat(parsed.length, is(1))
- }
-
- @Test
- def port_can_be_configured {
- val parsed = toConfigServers("myhost:123")
- val port: Int = parsed(0).port.get()
- assertThat(port, is(123))
- }
-
- @Test
- def multiple_spaces_are_supported {
- val parsed = toConfigServers("test1 test2")
- assertThat(parsed.size, is(2))
-
- val hostNames = parsed.map(_.hostName)
- assertThat(hostNames, arrayContaining("test1", "test2"))
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def missing_port_gives_exception {
- toConfigServer("myhost:")
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def non_numeric_port_gives_exception {
- toConfigServer("myhost:non-numeric")
- }
-
- @Test
- def string_arrays_are_split_on_spaces {
- val parsed = toConfigModelsPluginDir("/home/vespa/foo /home/vespa/bar ")
- assertThat(parsed.size, is(2))
- }
-
- @Test
- def string_arrays_are_split_on_comma {
- val parsed = toConfigModelsPluginDir("/home/vespa/foo,/home/vespa/bar,")
- assertThat(parsed.size, is(2))
- }
-}
diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala
deleted file mode 100644
index 33f9a2e8594..00000000000
--- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-import com.yahoo.config.model.producer.AbstractConfigProducerRoot
-import com.yahoo.config.model.test.MockRoot
-import com.yahoo.container.Container
-import com.yahoo.jdisc.test.TestDriver
-import scala.xml.Node
-import com.yahoo.vespa.model.VespaModel
-import com.yahoo.io.IOUtils
-import java.nio.file.{Files, Path}
-import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking
-
-/**
- * Creates a local application from vespa-services fragments.
- *
- * @author tonytv
- */
-object StandaloneContainer {
- def firstContainerId(root: AbstractConfigProducerRoot): String = {
- root.getConfigProducer("container").get().getConfigId
- }
-
- def withStandaloneContainer[T](containerNode: Node) {
- withTempDirectory { applicationDirectory =>
- System.setProperty(StandaloneContainerApplication.applicationLocationInstallVariable, applicationDirectory.toString)
- createServicesXml(applicationDirectory, containerNode)
-
- val driver = TestDriver.newInjectedApplicationInstance(classOf[StandaloneContainerApplication])
- driver.close()
- Container.resetInstance()
- }
- }
-
- def withContainerModel[T](containerNode: Node)(f: VespaModel => T) {
- withTempDirectory { applicationPath =>
- createServicesXml(applicationPath, containerNode)
-
- val distributedFiles = new LocalFileDb(applicationPath)
- val (root, container) = StandaloneContainerApplication.createContainerModel(
- applicationPath,
- distributedFiles,
- applicationPath.resolve("preprocesedApp").toFile,
- networkingOption = Networking.enable)
- f(root)
- }
- }
-
- private def withTempDirectory[T](f : Path => T) : T = {
- val directory = Files.createTempDirectory("application")
- try {
- f(directory)
- } finally {
- IOUtils.recursiveDeleteDir(directory.toFile)
- }
- }
-
- private def createServicesXml(applicationPath : Path,
- containerNode: Node) {
-
- scala.xml.XML.save(applicationPath.resolve("services.xml").toFile.getAbsolutePath,
- containerNode, xmlDecl = true)
- }
-}
diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala
deleted file mode 100644
index 87bef2efd95..00000000000
--- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-
-import com.yahoo.container.standalone.StandaloneContainerTest._
-import com.yahoo.vespa.model.AbstractService
-import org.junit.Assert._
-import org.junit.Test
-
-import scala.util.Try
-
-
-/**
- * @author tonytv
- * @author gjoranv
- */
-
-class StandaloneContainerTest {
- @Test
- def container_is_allowed_root_element() {
- StandaloneContainer.withContainerModel(plainXml) { root => }
- }
-
- @Test
- def services_is_allowed_root_element() {
- val servicesXml =
- <services>
- <container version="1.0" />
- </services>
-
- StandaloneContainer.withContainerModel(servicesXml) { root => }
- }
-
- @Test
- def multiple_container_elements_cannot_be_deployed() {
- val twoContainersXml =
- <services>
- <container id="container-1" version="1.0" />
- <container id="container-2" version="1.0" />
- </services>
-
- assertTrue(
- Try {
- StandaloneContainer.withContainerModel(twoContainersXml) { root => }
- }.isFailure)
- }
-
- @Test
- def application_preprocessor_is_run() {
- val servicesXml =
- <services xmlns:preprocess="properties">
- <preprocess:properties>
- <container_id>container-1</container_id>
- </preprocess:properties>
- <container id="${container_id}" version="1.0" />
- </services>
- StandaloneContainer.withContainerModel(servicesXml) {
- root =>
- assertTrue(root.getConfigProducer("container-1/standalone").isPresent)
- }
- }
-
- @Test
- def no_default_ports_are_enabled_when_using_http() {
- val xml =
- <jdisc version="1.0">
- <http>
- <server port="4000" id="server1" />
- </http>
- </jdisc>
-
- StandaloneContainer.withContainerModel(xml) { root =>
- val container = root.getConfigProducer("jdisc/standalone").get().asInstanceOf[AbstractService]
- println("portCnt: " + container.getPortCount)
- println("numPorts: " + container.getNumPortsAllocated)
- assertEquals(1, container.getNumPortsAllocated)
- }
- }
-
-}
-
-object StandaloneContainerTest {
-
- val plainXml = <container version="1.0" />
-}
diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala
deleted file mode 100644
index ab6f486c748..00000000000
--- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone
-
-import org.junit.{Ignore, Test}
-import org.junit.Assert.assertThat
-import org.hamcrest.CoreMatchers.is
-import org.hamcrest.number.OrderingComparison.greaterThan
-
-import StandaloneContainer.withContainerModel
-import com.yahoo.vespa.config.ConfigKey
-import com.yahoo.config.ConfigInstance
-import com.yahoo.container.{ComponentsConfig, BundlesConfig, di}
-import scala.collection.JavaConverters._
-
-/**
- * @author tonytv
- */
-class StandaloneSubscriberTest {
- val bundlesKey = key("bundles")
- val componentsKey = key("components")
-
- def key(name: String) = new ConfigKey(name, "container", "container").asInstanceOf[ConfigKey[ConfigInstance]]
-
- def box(i: Int) = java.lang.Integer.valueOf(i)
-
- @Test
- @Ignore
- def standalone_subscriber() {
- withContainerModel(<container version="1.0"> </container>) { root =>
- val subscriber = new StandaloneSubscriberFactory(root).getSubscriber(Set(bundlesKey, componentsKey).asJava)
- val config = subscriber.config.asScala
- assertThat(config.size, is(2))
-
- val bundlesConfig = config(bundlesKey).asInstanceOf[BundlesConfig]
- val componentsConfig = config(componentsKey).asInstanceOf[ComponentsConfig]
-
- assertThat(bundlesConfig.bundle().size(), is(0))
- assertThat(box(componentsConfig.components().size()), greaterThan(box(10)))
- }
- }
-}
diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp
index a271fbccf50..3b72585b076 100644
--- a/storage/src/tests/storageserver/communicationmanagertest.cpp
+++ b/storage/src/tests/storageserver/communicationmanagertest.cpp
@@ -14,6 +14,8 @@
#include <vespa/documentapi/messagebus/messages/getdocumentmessage.h>
#include <vespa/vdstestlib/cppunit/macros.h>
#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h>
+#include <vespa/documentapi/messagebus/messages/getdocumentreply.h>
using document::test::makeDocumentBucket;
@@ -27,6 +29,7 @@ struct CommunicationManagerTest : public CppUnit::TestFixture {
void testRepliesAreDequeuedInFifoOrder();
void bucket_space_config_can_be_updated_live();
void unmapped_bucket_space_documentapi_request_returns_error_reply();
+ void unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply();
static constexpr uint32_t MESSAGE_WAIT_TIME_SEC = 60;
@@ -51,6 +54,7 @@ struct CommunicationManagerTest : public CppUnit::TestFixture {
CPPUNIT_TEST(testRepliesAreDequeuedInFifoOrder);
CPPUNIT_TEST(bucket_space_config_can_be_updated_live);
CPPUNIT_TEST(unmapped_bucket_space_documentapi_request_returns_error_reply);
+ CPPUNIT_TEST(unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply);
CPPUNIT_TEST_SUITE_END();
};
@@ -267,13 +271,21 @@ struct CommunicationManagerFixture {
}
~CommunicationManagerFixture();
- std::unique_ptr<documentapi::GetDocumentMessage> documentapi_message_for_space(const char* space) {
- auto cmd = std::make_unique<documentapi::GetDocumentMessage>(
- document::DocumentId(vespalib::make_string("id::%s::stuff", space)));
+ template <typename T>
+ std::unique_ptr<T> documentapi_message_for_space(const char *space) {
+ auto cmd = std::make_unique<T>(document::DocumentId(vespalib::make_string("id::%s::stuff", space)));
// Bind reply handling to our own mock handler
cmd->pushHandler(reply_handler);
return cmd;
}
+
+ std::unique_ptr<documentapi::RemoveDocumentMessage> documentapi_remove_message_for_space(const char *space) {
+ return documentapi_message_for_space<documentapi::RemoveDocumentMessage>(space);
+ }
+
+ std::unique_ptr<documentapi::GetDocumentMessage> documentapi_get_message_for_space(const char *space) {
+ return documentapi_message_for_space<documentapi::GetDocumentMessage>(space);
+ }
};
CommunicationManagerFixture::~CommunicationManagerFixture() = default;
@@ -298,8 +310,8 @@ void CommunicationManagerTest::bucket_space_config_can_be_updated_live() {
config.documenttype.emplace_back(doc_type("bar", "global"));
f.comm_mgr->updateBucketSpacesConfig(config);
- f.comm_mgr->handleMessage(f.documentapi_message_for_space("bar"));
- f.comm_mgr->handleMessage(f.documentapi_message_for_space("foo"));
+ f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("bar"));
+ f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("foo"));
f.bottom_link->waitForMessages(2, MESSAGE_WAIT_TIME_SEC);
auto cmd1 = f.bottom_link->getCommand(0);
@@ -310,7 +322,7 @@ void CommunicationManagerTest::bucket_space_config_can_be_updated_live() {
config.documenttype[1] = doc_type("bar", "default");
f.comm_mgr->updateBucketSpacesConfig(config);
- f.comm_mgr->handleMessage(f.documentapi_message_for_space("bar"));
+ f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("bar"));
f.bottom_link->waitForMessages(3, MESSAGE_WAIT_TIME_SEC);
auto cmd3 = f.bottom_link->getCommand(2);
@@ -328,7 +340,7 @@ void CommunicationManagerTest::unmapped_bucket_space_documentapi_request_returns
CPPUNIT_ASSERT_EQUAL(uint64_t(0), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue());
- f.comm_mgr->handleMessage(f.documentapi_message_for_space("fluff"));
+ f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("fluff"));
CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size());
auto& reply = *f.reply_handler.replies[0];
CPPUNIT_ASSERT(reply.hasErrors());
@@ -337,4 +349,23 @@ void CommunicationManagerTest::unmapped_bucket_space_documentapi_request_returns
CPPUNIT_ASSERT_EQUAL(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue());
}
+// Legacy DocumentAPI routing protocols will send Gets to _all_ clusters even
+// if they do not contain a particular document type. By sending an empty reply
+// we signal a mergeable "not found" to the sender rather than a non-mergeable
+// fatal error.
+void CommunicationManagerTest::unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply() {
+ CommunicationManagerFixture f;
+
+ BucketspacesConfigBuilder config;
+ config.documenttype.emplace_back(doc_type("foo", "default"));
+ f.comm_mgr->updateBucketSpacesConfig(config);
+
+ f.comm_mgr->handleMessage(f.documentapi_get_message_for_space("fluff"));
+ CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size());
+ auto& reply = *f.reply_handler.replies[0];
+ CPPUNIT_ASSERT(!reply.hasErrors());
+ auto& get_reply = dynamic_cast<documentapi::GetDocumentReply&>(reply);
+ CPPUNIT_ASSERT(!get_reply.hasDocument());
+}
+
} // storage
diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
index a08445ca3d2..471ce7e2b27 100644
--- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
+++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp
@@ -236,6 +236,7 @@ PendingClusterState::onRequestBucketInfoReply(const std::shared_ptr<api::Request
if (result == api::ReturnCode::Result::ENCODE_ERROR) {
// Handle failure to encode bucket space due to use of old storage api
// protocol. Pretend that request succeeded with no buckets returned.
+ // TODO remove this workaround for Vespa 7
LOG(debug, "Got ENCODE_ERROR, pretending success with no buckets");
} else if (!result.success()) {
framework::MilliSecTime resendTime(_clock);
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
index 22193d7e246..94a151bcdc1 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
@@ -19,6 +19,7 @@
#include <vespa/log/bufferedlogger.h>
#include <vespa/document/bucket/fixed_bucket_spaces.h>
+#include <vespa/documentapi/messagebus/messages/getdocumentreply.h>
LOG_SETUP(".communication.manager");
@@ -258,8 +259,17 @@ void CommunicationManager::fail_with_unresolvable_bucket_space(
{
LOG(debug, "Could not map DocumentAPI message to internal bucket: %s", error_message.c_str());
MBUS_TRACE(msg->getTrace(), 6, "Communication manager: Failing message as its document type has no known bucket space mapping");
- std::unique_ptr<mbus::Reply> reply(new mbus::EmptyReply());
- reply->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_REJECTED, error_message));
+ std::unique_ptr<mbus::Reply> reply;
+ if (msg->getType() == documentapi::DocumentProtocol::MESSAGE_GETDOCUMENT) {
+ // HACK: to avoid breaking legacy routing of GetDocumentMessages to _all_ clusters
+ // regardless of them having a document type or not, we remap missing bucket spaces
+ // to explicit Not Found replies (empty document GetDocumentReply).
+ // TODO remove this workaround for Vespa 7
+ reply = std::make_unique<documentapi::GetDocumentReply>(std::shared_ptr<document::Document>());
+ } else {
+ reply = std::make_unique<mbus::EmptyReply>();
+ reply->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_REJECTED, error_message));
+ }
msg->swapState(*reply);
_metrics.bucketSpaceMappingFailures.inc();
_messageBusSession->reply(std::move(reply));
diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp
index ad98d64b173..6bb2ca31ec1 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenode.cpp
@@ -206,10 +206,8 @@ StorageNode::initialize()
_chain.reset(createChain().release());
- if (_component->enableMultipleBucketSpaces()) {
- assert(_communicationManager != nullptr);
- _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig);
- }
+ assert(_communicationManager != nullptr);
+ _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig);
// Start the metric manager, such that it starts generating snapshots
// and the like. Note that at this time, all metrics should hopefully
@@ -359,9 +357,7 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard)
if (_newBucketSpacesConfig) {
_bucketSpacesConfig = std::move(_newBucketSpacesConfig);
_context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig);
- if (_component->enableMultipleBucketSpaces()) {
- _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig);
- }
+ _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig);
}
}
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp
index f1e96cc1631..a1be0def20b 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp
@@ -60,8 +60,7 @@ ProtocolSerialization5_0::getBucketInfo(document::ByteBuffer& buf) const
}
void
-ProtocolSerialization5_0::putBucketInfo(
- const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const
+ProtocolSerialization5_0::putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const
{
buf.putInt(info.getChecksum());
buf.putInt(info.getDocumentCount());
@@ -71,8 +70,7 @@ ProtocolSerialization5_0::putBucketInfo(
}
void
-ProtocolSerialization5_0::onEncodeReply(
- GBBuf& buf, const api::StorageReply& msg) const
+ProtocolSerialization5_0::onEncodeReply(GBBuf& buf, const api::StorageReply& msg) const
{
SH::putReturnCode(msg.getResult(), buf);
buf.putLong(msg.getMsgId());
@@ -80,8 +78,7 @@ ProtocolSerialization5_0::onEncodeReply(
}
void
-ProtocolSerialization5_0::onDecodeReply(BBuf& buf,
- api::StorageReply& msg) const
+ProtocolSerialization5_0::onDecodeReply(BBuf& buf, api::StorageReply& msg) const
{
msg.setResult(SH::getReturnCode(buf));
msg.forceMsgId(SH::getLong(buf));
@@ -89,8 +86,7 @@ ProtocolSerialization5_0::onDecodeReply(BBuf& buf,
}
void
-ProtocolSerialization5_0::onEncodeCommand(
- GBBuf& buf, const api::StorageCommand& msg) const
+ProtocolSerialization5_0::onEncodeCommand(GBBuf& buf, const api::StorageCommand& msg) const
{
buf.putLong(msg.getMsgId());
buf.putByte(msg.getPriority());
@@ -99,8 +95,7 @@ ProtocolSerialization5_0::onEncodeCommand(
}
void
-ProtocolSerialization5_0::onDecodeCommand(BBuf& buf,
- api::StorageCommand& msg) const
+ProtocolSerialization5_0::onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const
{
msg.forceMsgId(SH::getLong(buf));
uint8_t priority = SH::getByte(buf);
@@ -118,15 +113,13 @@ ProtocolSerialization5_0::ProtocolSerialization5_0(
{
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::PutReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::PutReply& msg) const
{
buf.putBoolean(msg.wasFound());
onEncodeBucketInfoReply(buf, msg);
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::PutCommand& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::PutCommand& msg) const
{
SH::putDocument(msg.getDocument().get(), buf);
putBucket(msg.getBucket(), buf);
@@ -141,24 +134,22 @@ ProtocolSerialization5_0::onDecodePutCommand(BBuf& buf) const
document::Document::SP doc(SH::getDocument(buf, getTypeRepo()));
document::Bucket bucket = getBucket(buf);
api::Timestamp ts(SH::getLong(buf));
- api::PutCommand::UP msg(new api::PutCommand(bucket, doc, ts));
+ auto msg = std::make_unique<api::PutCommand>(bucket, doc, ts);
msg->setUpdateTimestamp(SH::getLong(buf));
onDecodeBucketInfoCommand(buf, *msg);
- return api::StorageCommand::UP(msg.release());
+ return msg;
}
api::StorageReply::UP
ProtocolSerialization5_0::onDecodePutReply(const SCmd& cmd, BBuf& buf) const
{
bool wasFound = SH::getBoolean(buf);
- api::PutReply::UP msg(new api::PutReply(
- static_cast<const api::PutCommand&>(cmd), wasFound));
+ auto msg = std::make_unique<api::PutReply>(static_cast<const api::PutCommand&>(cmd), wasFound);
onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::UpdateReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::UpdateReply& msg) const
{
buf.putLong(msg.getOldTimestamp());
onEncodeBucketInfoReply(buf, msg);
@@ -168,14 +159,12 @@ api::StorageReply::UP
ProtocolSerialization5_0::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const
{
api::Timestamp oldTimestamp(SH::getLong(buf));
- api::UpdateReply::UP msg(new api::UpdateReply(
- static_cast<const api::UpdateCommand&>(cmd), oldTimestamp));
+ auto msg = std::make_unique<api::UpdateReply>(static_cast<const api::UpdateCommand&>(cmd), oldTimestamp);
onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::GetReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetReply& msg) const
{
SH::putDocument(msg.getDocument().get(), buf);
buf.putLong(msg.getLastModifiedTimestamp());
@@ -188,28 +177,17 @@ ProtocolSerialization5_0::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const
try {
document::Document::SP doc(SH::getDocument(buf, getTypeRepo()));
api::Timestamp lastModified(SH::getLong(buf));
- api::GetReply::UP msg(
- new api::GetReply(
- static_cast<const api::GetCommand&>(cmd),
- doc,
- lastModified));
+ auto msg = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), doc,lastModified);
onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
} catch (std::exception& e) {
- api::GetReply::UP msg(
- new api::GetReply(
- static_cast<const api::GetCommand&>(cmd),
- document::Document::SP(),
- 0));
- msg->setResult(api::ReturnCode(
- api::ReturnCode::UNPARSEABLE,
- e.what()));
- return api::StorageReply::UP(msg.release());
+ auto msg = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), document::Document::SP(),0);
+ msg->setResult(api::ReturnCode(api::ReturnCode::UNPARSEABLE, e.what()));
+ return msg;
}
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::RemoveReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RemoveReply& msg) const
{
buf.putLong(msg.getOldTimestamp());
onEncodeBucketInfoReply(buf, msg);
@@ -219,14 +197,12 @@ api::StorageReply::UP
ProtocolSerialization5_0::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const
{
api::Timestamp oldTimestamp(SH::getLong(buf));
- api::RemoveReply::UP msg(new api::RemoveReply(
- static_cast<const api::RemoveCommand&>(cmd), oldTimestamp));
+ auto msg = std::make_unique<api::RemoveReply>(static_cast<const api::RemoveCommand&>(cmd), oldTimestamp);
onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::UpdateCommand& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::UpdateCommand& msg) const
{
document::DocumentUpdate* update = msg.getUpdate().get();
if (update) {
@@ -253,24 +229,19 @@ ProtocolSerialization5_0::onDecodeUpdateCommand(BBuf& buf) const
if (size != 0) {
document::ByteBuffer bbuf(buf.getBufferAtPos(), size);
buf.incPos(size);
- update.reset(new document::DocumentUpdate(getTypeRepo(), bbuf,
- document::DocumentUpdate::
- SerializeVersion::
- SERIALIZE_HEAD));
+ update = document::DocumentUpdate::createHEAD(getTypeRepo(), bbuf);
}
document::Bucket bucket = getBucket(buf);
api::Timestamp timestamp(SH::getLong(buf));
- api::UpdateCommand::UP msg(
- new api::UpdateCommand(bucket, update, timestamp));
+ api::UpdateCommand::UP msg = std::make_unique<api::UpdateCommand>(bucket, update, timestamp);
msg->setOldTimestamp(SH::getLong(buf));
onDecodeBucketInfoCommand(buf, *msg);
- return api::StorageCommand::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::RevertReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RevertReply& msg) const
{
onEncodeBucketInfoReply(buf, msg);
}
@@ -278,31 +249,26 @@ void ProtocolSerialization5_0::onEncode(
api::StorageReply::UP
ProtocolSerialization5_0::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const
{
- api::RevertReply::UP msg(new api::RevertReply(
- static_cast<const api::RevertCommand&>(cmd)));
+ auto msg = std::make_unique<api::RevertReply>(static_cast<const api::RevertCommand&>(cmd));
onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::CreateBucketReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateBucketReply& msg) const
{
onEncodeBucketInfoReply(buf, msg);
}
api::StorageReply::UP
-ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd,
- BBuf& buf) const
+ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd, BBuf& buf) const
{
- api::CreateBucketReply::UP msg(new api::CreateBucketReply(
- static_cast<const api::CreateBucketCommand&>(cmd)));
+ auto msg = std::make_unique<api::CreateBucketReply>(static_cast<const api::CreateBucketCommand&>(cmd));
onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
void
-ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::DeleteBucketCommand& msg) const
+ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::DeleteBucketCommand& msg) const
{
putBucket(msg.getBucket(), buf);
onEncodeBucketInfoCommand(buf, msg);
@@ -313,32 +279,28 @@ api::StorageCommand::UP
ProtocolSerialization5_0::onDecodeDeleteBucketCommand(BBuf& buf) const
{
document::Bucket bucket = getBucket(buf);
- api::DeleteBucketCommand::UP msg(new api::DeleteBucketCommand(bucket));
+ auto msg = std::make_unique<api::DeleteBucketCommand>(bucket);
onDecodeBucketInfoCommand(buf, *msg);
if (buf.getRemaining() >= SH::BUCKET_INFO_SERIALIZED_SIZE) {
msg->setBucketInfo(getBucketInfo(buf));
}
- return api::StorageCommand::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::DeleteBucketReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::DeleteBucketReply& msg) const
{
onEncodeBucketInfoReply(buf, msg);
}
api::StorageReply::UP
-ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd,
- BBuf& buf) const
+ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd, BBuf& buf) const
{
- api::DeleteBucketReply::UP msg(new api::DeleteBucketReply(
- static_cast<const api::DeleteBucketCommand&>(cmd)));
+ auto msg = std::make_unique<api::DeleteBucketReply>(static_cast<const api::DeleteBucketCommand&>(cmd));
onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::MergeBucketCommand& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const
{
ProtocolSerialization4_2::onEncode(buf, msg);
@@ -353,8 +315,7 @@ void ProtocolSerialization5_0::onEncode(
api::StorageCommand::UP
ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const
{
- api::StorageCommand::UP cmd
- = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf);
+ api::StorageCommand::UP cmd = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf);
uint32_t clusterStateVersion = SH::getInt(buf);
uint16_t chainSize = SH::getShort(buf);
std::vector<uint16_t> chain;
@@ -369,24 +330,20 @@ ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const
return cmd;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::MergeBucketReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::MergeBucketReply& msg) const
{
onEncodeBucketReply(buf, msg);
}
api::StorageReply::UP
-ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd,
- BBuf& buf) const
+ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd, BBuf& buf) const
{
- api::MergeBucketReply::UP msg(new api::MergeBucketReply(
- static_cast<const api::MergeBucketCommand&>(cmd)));
+ auto msg = std::make_unique<api::MergeBucketReply>(static_cast<const api::MergeBucketCommand&>(cmd));
onDecodeBucketReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::GetBucketDiffReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetBucketDiffReply& msg) const
{
const std::vector<api::GetBucketDiffCommand::Entry>& entries(msg.getDiff());
buf.putInt(entries.size());
@@ -397,11 +354,9 @@ void ProtocolSerialization5_0::onEncode(
}
api::StorageReply::UP
-ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd,
- BBuf& buf) const
+ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd, BBuf& buf) const
{
- api::GetBucketDiffReply::UP msg(new api::GetBucketDiffReply(
- static_cast<const api::GetBucketDiffCommand&>(cmd)));
+ auto msg = std::make_unique<api::GetBucketDiffReply>(static_cast<const api::GetBucketDiffCommand&>(cmd));
std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff());
uint32_t entryCount = SH::getInt(buf);
if (entryCount > buf.getRemaining()) {
@@ -413,24 +368,20 @@ ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd,
onDecodeDiffEntry(buf, entries[i]);
}
onDecodeBucketReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::ApplyBucketDiffReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::ApplyBucketDiffReply& msg) const
{
- const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(
- msg.getDiff());
+ const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg.getDiff());
buf.putInt(entries.size());
for (uint32_t i=0; i<entries.size(); ++i) {
onEncodeDiffEntry(buf, entries[i]._entry);
buf.putString(entries[i]._docName);
buf.putInt(entries[i]._headerBlob.size());
- buf.putBytes(&entries[i]._headerBlob[0],
- entries[i]._headerBlob.size());
+ buf.putBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size());
buf.putInt(entries[i]._bodyBlob.size());
- buf.putBytes(&entries[i]._bodyBlob[0],
- entries[i]._bodyBlob.size());
+ buf.putBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size());
}
onEncodeBucketInfoReply(buf, msg);
}
@@ -439,8 +390,7 @@ api::StorageReply::UP
ProtocolSerialization5_0::onDecodeApplyBucketDiffReply(const SCmd& cmd,
BBuf& buf) const
{
- api::ApplyBucketDiffReply::UP msg(new api::ApplyBucketDiffReply(
- static_cast<const api::ApplyBucketDiffCommand&>(cmd)));
+ auto msg = std::make_unique<api::ApplyBucketDiffReply>(static_cast<const api::ApplyBucketDiffCommand&>(cmd));
std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff());
uint32_t entryCount = SH::getInt(buf);
if (entryCount > buf.getRemaining()) {
@@ -456,25 +406,21 @@ ProtocolSerialization5_0::onDecodeApplyBucketDiffReply(const SCmd& cmd,
buf.incPos(headerSize);
}
entries[i]._headerBlob.resize(headerSize);
- buf.getBytes(&entries[i]._headerBlob[0],
- entries[i]._headerBlob.size());
+ buf.getBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size());
uint32_t bodySize = SH::getInt(buf);
if (bodySize > buf.getRemaining()) {
buf.incPos(bodySize);
}
entries[i]._bodyBlob.resize(bodySize);
- buf.getBytes(&entries[i]._bodyBlob[0],
- entries[i]._bodyBlob.size());
+ buf.getBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size());
}
onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::SplitBucketReply& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::SplitBucketReply& msg) const
{
- const std::vector<api::SplitBucketReply::Entry>& entries(
- msg.getSplitInfo());
+ const std::vector<api::SplitBucketReply::Entry>& entries(msg.getSplitInfo());
buf.putInt(entries.size());
for (std::vector<api::SplitBucketReply::Entry>::const_iterator it
= entries.begin(); it != entries.end(); ++it)
@@ -486,11 +432,9 @@ void ProtocolSerialization5_0::onEncode(
}
api::StorageReply::UP
-ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd,
- BBuf& buf) const
+ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, BBuf& buf) const
{
- api::SplitBucketReply::UP msg(new api::SplitBucketReply(
- static_cast<const api::SplitBucketCommand&>(cmd)));
+ auto msg = std::make_unique<api::SplitBucketReply>(static_cast<const api::SplitBucketCommand&>(cmd));
std::vector<api::SplitBucketReply::Entry>& entries(msg->getSplitInfo());
uint32_t targetCount = SH::getInt(buf);
if (targetCount > buf.getRemaining()) {
@@ -505,12 +449,11 @@ ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd,
it->second = getBucketInfo(buf);
}
onDecodeBucketReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
void
-ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::JoinBucketsCommand& msg) const
+ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::JoinBucketsCommand& msg) const
{
putBucket(msg.getBucket(), buf);
buf.putInt(msg.getSourceBuckets().size());
@@ -525,7 +468,7 @@ api::StorageCommand::UP
ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const
{
document::Bucket bucket = getBucket(buf);
- api::JoinBucketsCommand::UP msg(new api::JoinBucketsCommand(bucket));
+ auto msg = std::make_unique<api::JoinBucketsCommand>(bucket);
uint32_t size = SH::getInt(buf);
if (size > buf.getRemaining()) {
// Trigger out of bounds exception rather than out of memory error
@@ -537,55 +480,48 @@ ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const
}
msg->setMinJoinBits(SH::getByte(buf));
onDecodeCommand(buf, *msg);
- return api::StorageCommand::UP(msg.release());
+ return msg;
}
void
-ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::JoinBucketsReply& msg) const
+ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::JoinBucketsReply& msg) const
{
putBucketInfo(msg.getBucketInfo(), buf);
onEncodeBucketReply(buf, msg);
}
api::StorageReply::UP
-ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd,
- BBuf& buf) const
+ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const
{
- api::JoinBucketsReply::UP msg(new api::JoinBucketsReply(
- static_cast<const api::JoinBucketsCommand&>(cmd)));
+ auto msg = std::make_unique<api::JoinBucketsReply>(static_cast<const api::JoinBucketsCommand&>(cmd));
msg->setBucketInfo(getBucketInfo(buf));
onDecodeBucketReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
void
-ProtocolSerialization5_0::onEncodeBucketInfoReply(
- GBBuf& buf, const api::BucketInfoReply& msg) const
+ProtocolSerialization5_0::onEncodeBucketInfoReply(GBBuf& buf, const api::BucketInfoReply& msg) const
{
onEncodeBucketReply(buf, msg);
putBucketInfo(msg.getBucketInfo(), buf);
}
void
-ProtocolSerialization5_0::onDecodeBucketInfoReply(
- BBuf& buf, api::BucketInfoReply& msg) const
+ProtocolSerialization5_0::onDecodeBucketInfoReply(BBuf& buf, api::BucketInfoReply& msg) const
{
onDecodeBucketReply(buf, msg);
msg.setBucketInfo(getBucketInfo(buf));
}
void
-ProtocolSerialization5_0::onEncodeBucketReply(
- GBBuf& buf, const api::BucketReply& msg) const
+ProtocolSerialization5_0::onEncodeBucketReply(GBBuf& buf, const api::BucketReply& msg) const
{
onEncodeReply(buf, msg);
buf.putLong(msg.hasBeenRemapped() ? msg.getBucketId().getRawId() : 0);
}
void
-ProtocolSerialization5_0::onDecodeBucketReply(
- BBuf& buf, api::BucketReply& msg) const
+ProtocolSerialization5_0::onDecodeBucketReply(BBuf& buf, api::BucketReply& msg) const
{
onDecodeReply(buf, msg);
document::BucketId bucket(SH::getLong(buf));
@@ -608,11 +544,9 @@ ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateVisitorReply& ms
}
api::StorageReply::UP
-ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd,
- BBuf& buf) const
+ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const
{
- api::CreateVisitorReply::UP msg(new api::CreateVisitorReply(
- static_cast<const api::CreateVisitorCommand&>(cmd)));
+ auto msg = std::make_unique<api::CreateVisitorReply>(static_cast<const api::CreateVisitorCommand&>(cmd));
onDecodeReply(buf, *msg);
vdslib::VisitorStatistics vs;
@@ -625,11 +559,10 @@ ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd,
vs.setSecondPassBytesReturned(SH::getLong(buf));
msg->setVisitorStatistics(vs);
- return api::StorageReply::UP(msg.release());
+ return msg;
}
-void ProtocolSerialization5_0::onEncode(
- GBBuf& buf, const api::RequestBucketInfoCommand& msg) const
+void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RequestBucketInfoCommand& msg) const
{
const std::vector<document::BucketId>& buckets(msg.getBuckets());
buf.putInt(buckets.size());
@@ -663,7 +596,7 @@ ProtocolSerialization5_0::onDecodeRequestBucketInfoCommand(BBuf& buf) const
msg.reset(new api::RequestBucketInfoCommand(bucketSpace, distributor, state, SH::getString(buf)));
}
onDecodeCommand(buf, *msg);
- return api::StorageCommand::UP(msg.release());
+ return msg;
}
void
diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h
index 02c4c0ce0b9..08cec601cce 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h
+++ b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h
@@ -4,13 +4,11 @@
#include <vespa/fastos/types.h>
#include <vespa/document/base/globalid.h>
#include <vespa/document/fieldvalue/document.h>
-#include <vespa/document/update/documentupdate.h>
#include <vespa/document/util/bytebuffer.h>
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/util/growablebytebuffer.h>
-namespace storage {
-namespace mbusprot {
+namespace storage::mbusprot {
class SerializationHelper
{
@@ -60,8 +58,7 @@ public:
return api::ReturnCode(result, message);
}
- static void putReturnCode(const api::ReturnCode& code,
- vespalib::GrowableByteBuffer& buf)
+ static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf)
{
buf.putInt(code.getResult());
buf.putString(code.getMessage());
@@ -77,18 +74,14 @@ public:
return document::GlobalId(&buffer[0]);
}
- static void putGlobalId(const document::GlobalId& gid,
- vespalib::GrowableByteBuffer& buf)
+ static void putGlobalId(const document::GlobalId& gid, vespalib::GrowableByteBuffer& buf)
{
buf.putShort(document::GlobalId::LENGTH);
for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) {
buf.putByte(gid.get()[i]);
}
}
-
- static document::Document::UP getDocument(
- document::ByteBuffer& buf,
- const document::DocumentTypeRepo& repo)
+ static document::Document::UP getDocument(document::ByteBuffer& buf, const document::DocumentTypeRepo& repo)
{
uint32_t size = getInt(buf);
if (size == 0) {
@@ -100,26 +93,7 @@ public:
}
}
- static document::DocumentUpdate::UP getUpdate(
- document::ByteBuffer& buf,
- const document::DocumentTypeRepo& repo)
- {
- uint32_t size = getInt(buf);
- if (size == 0) {
- return document::DocumentUpdate::UP();
- } else {
- document::ByteBuffer bbuf(buf.getBufferAtPos(), size);
- buf.incPos(size);
- return document::DocumentUpdate::UP(
- new document::DocumentUpdate(repo, bbuf,
- document::DocumentUpdate::
- SerializeVersion::
- SERIALIZE_42));
- }
- }
-
- static void putDocument(document::Document* doc,
- vespalib::GrowableByteBuffer& buf)
+ static void putDocument(document::Document* doc, vespalib::GrowableByteBuffer& buf)
{
if (doc) {
vespalib::nbostream stream;
@@ -131,21 +105,6 @@ public:
}
}
- static void putUpdate(document::DocumentUpdate* update,
- vespalib::GrowableByteBuffer& buf)
- {
- if (update) {
- vespalib::nbostream stream;
- update->serialize42(stream);
- buf.putInt(stream.size());
- buf.putBytes(stream.peek(), stream.size());
- } else {
- buf.putInt(0);
- }
- }
-
};
-} // mbusprot
-} // storage
-
+}
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
index 33fca0f161a..f83188f7dd8 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
@@ -17,12 +17,12 @@ mbus::string StorageProtocol::NAME = "StorageProtocol";
StorageProtocol::StorageProtocol(const std::shared_ptr<const document::DocumentTypeRepo> repo,
const documentapi::LoadTypeSet& loadTypes,
- bool activateBucketSpaceSerialization)
+ bool configForcedBucketSpaceSerialization)
: _serializer5_0(repo, loadTypes),
_serializer5_1(repo, loadTypes),
_serializer5_2(repo, loadTypes),
_serializer6_0(repo, loadTypes),
- _activateBucketSpaceSerialization(activateBucketSpaceSerialization)
+ _configForcedBucketSpaceSerialization(configForcedBucketSpaceSerialization)
{
}
@@ -106,7 +106,7 @@ StorageProtocol::encode(const vespalib::Version& version,
} else if (version < version5_2) {
return encodeMessage(_serializer5_1, routable, message, version5_1, version);
} else {
- if (_activateBucketSpaceSerialization) {
+ if (_configForcedBucketSpaceSerialization) {
return encodeMessage(_serializer6_0, routable, message, version6_0, version);
} else {
if (version < version6_0) {
@@ -184,7 +184,7 @@ StorageProtocol::decode(const vespalib::Version & version,
} else if (version < version5_2) {
return decodeMessage(_serializer5_1, data, type, version5_1, version);
} else {
- if (_activateBucketSpaceSerialization) {
+ if (_configForcedBucketSpaceSerialization) {
return decodeMessage(_serializer6_0, data, type, version6_0, version);
} else {
if (version < version6_0) {
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h
index d85e9d55d1a..56f271db1d0 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h
+++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h
@@ -29,7 +29,7 @@ private:
ProtocolSerialization5_1 _serializer5_1;
ProtocolSerialization5_2 _serializer5_2;
ProtocolSerialization6_0 _serializer6_0;
- bool _activateBucketSpaceSerialization;
+ bool _configForcedBucketSpaceSerialization;
};
}
diff --git a/storageserver/src/tests/testhelper.cpp b/storageserver/src/tests/testhelper.cpp
index b245a6500bd..5e6a71b078f 100644
--- a/storageserver/src/tests/testhelper.cpp
+++ b/storageserver/src/tests/testhelper.cpp
@@ -96,7 +96,11 @@ vdstestlib::DirConfig getStandardConfig(bool storagenode) {
// By default, need "old" behaviour of maxconcurrent
config->set("maxconcurrentvisitors_fixed", "4");
config->set("maxconcurrentvisitors_variable", "0");
- config = &dc.addConfig("stor-visitordispatcher");
+ dc.addConfig("stor-visitordispatcher");
+ config = &dc.addConfig("bucketspaces");
+ config->set("documenttype[1]");
+ config->set("documenttype[0].name", "testdoctype1");
+ config->set("documenttype[0].bucketspace", "default");
addFileConfig(dc, "documenttypes", "config-doctypes.cfg");
addStorageDistributionConfig(dc);
return dc;
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java
index 1504119d9cc..12389712976 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java
@@ -12,6 +12,8 @@ import com.yahoo.vespa.athenz.utils.AthenzIdentities;
import java.util.Base64;
+import static com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId.*;
+
/**
* Utility class for mapping objects model types and their Jackson binding versions.
*
@@ -33,7 +35,7 @@ public class EntityBindingsMapper {
public static VespaUniqueInstanceId toVespaUniqueInstanceId(VespaUniqueInstanceIdEntity entity) {
return new VespaUniqueInstanceId(
- entity.clusterIndex, entity.clusterId, entity.instance, entity.application, entity.tenant, entity.region, entity.environment);
+ entity.clusterIndex, entity.clusterId, entity.instance, entity.application, entity.tenant, entity.region, entity.environment, entity.type != null ? IdentityType.fromId(entity.type) : null); // TODO Remove support for legacy representation without type
}
public static IdentityDocument toIdentityDocument(IdentityDocumentEntity entity) {
@@ -50,17 +52,22 @@ public class EntityBindingsMapper {
toIdentityDocument(entity.identityDocument),
entity.signature,
entity.signingKeyVersion,
- VespaUniqueInstanceId.fromDottedString(entity.providerUniqueId),
+ fromDottedString(entity.providerUniqueId),
entity.dnsSuffix,
(AthenzService) AthenzIdentities.from(entity.providerService),
entity.ztsEndpoint,
- entity.documentVersion);
+ entity.documentVersion,
+ entity.configServerHostname,
+ entity.instanceHostname,
+ entity.createdAt,
+ entity.ipAddresses,
+ entity.identityType != null ? IdentityType.fromId(entity.identityType) : null); // TODO Remove support for legacy representation without type
}
public static VespaUniqueInstanceIdEntity toVespaUniqueInstanceIdEntity(VespaUniqueInstanceId model) {
return new VespaUniqueInstanceIdEntity(
model.tenant(), model.application(), model.environment(), model.region(),
- model.instance(), model.clusterId(), model.clusterIndex());
+ model.instance(), model.clusterId(), model.clusterIndex(), model.type() != null ? model.type().id() : null); // TODO Remove support for legacy representation without type
}
public static IdentityDocumentEntity toIdentityDocumentEntity(IdentityDocument model) {
@@ -84,7 +91,12 @@ public class EntityBindingsMapper {
model.dnsSuffix(),
model.providerService().getFullName(),
model.ztsEndpoint(),
- model.documentVersion());
+ model.documentVersion(),
+ model.configServerHostname(),
+ model.instanceHostname(),
+ model.createdAt(),
+ model.ipAddresses(),
+ model.identityType() != null ? model.identityType().id() : null); // TODO Remove support for legacy representation without type
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java
index 8da2bd0a343..82d0a3d622c 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java
@@ -8,7 +8,9 @@ import java.util.Set;
* The identity document that contains the instance specific information
*
* @author bjorncs
+ * @deprecated Will soon be inlined into {@link SignedIdentityDocument}
*/
+@Deprecated
public class IdentityDocument {
private final VespaUniqueInstanceId providerUniqueId;
private final String configServerHostname;
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityType.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityType.java
new file mode 100644
index 00000000000..4ca2e34a618
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityType.java
@@ -0,0 +1,25 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.identityprovider.api;
+
+import java.util.Arrays;
+
+/**
+ * Represents the types of identities that the configserver can provide.
+ *
+ * @author bjorncs
+ */
+public enum IdentityType {TENANT("tenant"), NODE("node");
+ private final String id;
+
+ IdentityType(String id) { this.id = id; }
+
+ public String id() { return id; }
+
+ public static IdentityType fromId(String id) {
+ return Arrays.stream(values())
+ .filter(v -> v.id.equals(id))
+ .findFirst()
+ .orElseThrow(() -> new IllegalArgumentException("Invalid id: " + id));
+ }
+}
+
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java
index d184efc0221..60be42544c7 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.athenz.identityprovider.api;
import com.yahoo.vespa.athenz.api.AthenzService;
import java.net.URI;
+import java.time.Instant;
+import java.util.Set;
/**
* A signed identity document which contains a {@link IdentityDocument}
@@ -22,6 +24,11 @@ public class SignedIdentityDocument {
private final AthenzService providerService;
private final URI ztsEndpoint;
private final int documentVersion;
+ private final String configServerHostname;
+ private final String instanceHostname;
+ private final Instant createdAt;
+ private final Set<String> ipAddresses;
+ private final IdentityType identityType;
public SignedIdentityDocument(IdentityDocument identityDocument,
String signature,
@@ -30,7 +37,12 @@ public class SignedIdentityDocument {
String dnsSuffix,
AthenzService providerService,
URI ztsEndpoint,
- int documentVersion) {
+ int documentVersion,
+ String configServerHostname,
+ String instanceHostname,
+ Instant createdAt,
+ Set<String> ipAddresses,
+ IdentityType identityType) {
this.identityDocument = identityDocument;
this.signature = signature;
this.signingKeyVersion = signingKeyVersion;
@@ -39,6 +51,11 @@ public class SignedIdentityDocument {
this.providerService = providerService;
this.ztsEndpoint = ztsEndpoint;
this.documentVersion = documentVersion;
+ this.configServerHostname = configServerHostname;
+ this.instanceHostname = instanceHostname;
+ this.createdAt = createdAt;
+ this.ipAddresses = ipAddresses;
+ this.identityType = identityType;
}
public IdentityDocument identityDocument() {
@@ -72,4 +89,24 @@ public class SignedIdentityDocument {
public int documentVersion() {
return documentVersion;
}
+
+ public String configServerHostname() {
+ return configServerHostname;
+ }
+
+ public String instanceHostname() {
+ return instanceHostname;
+ }
+
+ public Instant createdAt() {
+ return createdAt;
+ }
+
+ public Set<String> ipAddresses() {
+ return ipAddresses;
+ }
+
+ public IdentityType identityType() {
+ return identityType;
+ }
}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java
index 5539ba53882..be94cc59691 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.athenz.identityprovider.api;
import java.util.Objects;
/**
+ * Represents the unique instance id as used in Vespa's integration with Athenz Copper Argos
+ *
* @author bjorncs
*/
public class VespaUniqueInstanceId {
@@ -15,6 +17,7 @@ public class VespaUniqueInstanceId {
private final String tenant;
private final String region;
private final String environment;
+ private final IdentityType type;
public VespaUniqueInstanceId(int clusterIndex,
String clusterId,
@@ -22,7 +25,8 @@ public class VespaUniqueInstanceId {
String application,
String tenant,
String region,
- String environment) {
+ String environment,
+ IdentityType type) {
this.clusterIndex = clusterIndex;
this.clusterId = clusterId;
this.instance = instance;
@@ -30,21 +34,43 @@ public class VespaUniqueInstanceId {
this.tenant = tenant;
this.region = region;
this.environment = environment;
+ this.type = type;
}
+ // TODO Remove support for legacy representation without type
+ @Deprecated
+ public VespaUniqueInstanceId(int clusterIndex,
+ String clusterId,
+ String instance,
+ String application,
+ String tenant,
+ String region,
+ String environment) {
+ this(clusterIndex, clusterId, instance, application, tenant, region, environment, null);
+ }
+
+
+ // TODO Remove support for legacy representation without type
public static VespaUniqueInstanceId fromDottedString(String instanceId) {
String[] tokens = instanceId.split("\\.");
- if (tokens.length != 7) {
+ if (tokens.length != 7 && tokens.length != 8) {
throw new IllegalArgumentException("Invalid instance id: " + instanceId);
}
return new VespaUniqueInstanceId(
- Integer.parseInt(tokens[0]), tokens[1], tokens[2], tokens[3], tokens[4], tokens[5], tokens[6]);
+ Integer.parseInt(tokens[0]), tokens[1], tokens[2], tokens[3], tokens[4], tokens[5], tokens[6], tokens.length == 8 ? IdentityType.fromId(tokens[7]) : null);
}
+ // TODO Remove support for legacy representation without type
public String asDottedString() {
- return String.format(
- "%d.%s.%s.%s.%s.%s.%s",
- clusterIndex, clusterId, instance, application, tenant, region, environment);
+ if (type != null) {
+ return String.format(
+ "%d.%s.%s.%s.%s.%s.%s.%s",
+ clusterIndex, clusterId, instance, application, tenant, region, environment, type.id());
+ } else {
+ return String.format(
+ "%d.%s.%s.%s.%s.%s.%s",
+ clusterIndex, clusterId, instance, application, tenant, region, environment);
+ }
}
public int clusterIndex() {
@@ -75,6 +101,8 @@ public class VespaUniqueInstanceId {
return environment;
}
+ public IdentityType type() { return type; }
+
@Override
public String toString() {
return "VespaUniqueInstanceId{" +
@@ -85,6 +113,7 @@ public class VespaUniqueInstanceId {
", tenant='" + tenant + '\'' +
", region='" + region + '\'' +
", environment='" + environment + '\'' +
+ ", type=" + type +
'}';
}
@@ -99,11 +128,12 @@ public class VespaUniqueInstanceId {
Objects.equals(application, that.application) &&
Objects.equals(tenant, that.tenant) &&
Objects.equals(region, that.region) &&
- Objects.equals(environment, that.environment);
+ Objects.equals(environment, that.environment) &&
+ type == that.type;
}
@Override
public int hashCode() {
- return Objects.hash(clusterIndex, clusterId, instance, application, tenant, region, environment);
+ return Objects.hash(clusterIndex, clusterId, instance, application, tenant, region, environment, type);
}
}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java
index 775a49349a3..fc5392411c1 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java
@@ -5,7 +5,6 @@ import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
/**
@@ -16,11 +15,6 @@ public interface IdentityDocumentApi {
@GET
@Produces(MediaType.APPLICATION_JSON)
- @Deprecated
- SignedIdentityDocumentEntity getIdentityDocument(@QueryParam("hostname") String hostname);
-
- @GET
- @Produces(MediaType.APPLICATION_JSON)
@Path("/node/{host}")
SignedIdentityDocumentEntity getNodeIdentityDocument(@PathParam("host") String host);
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java
index 58a4f1e24bf..b4b2e82ab0e 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java
@@ -10,8 +10,10 @@ import java.util.Set;
/**
* @author bjorncs
+ * @deprecated Will soon be inlined into {@link SignedIdentityDocumentEntity}
*/
@JsonIgnoreProperties(ignoreUnknown = true)
+@Deprecated
public class IdentityDocumentEntity {
@JsonProperty("provider-unique-id")
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java
index e397b81ef9e..aa514b3caf3 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java
@@ -11,8 +11,10 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
+import java.time.Instant;
import java.util.Base64;
import java.util.Objects;
+import java.util.Set;
/**
* @author bjorncs
@@ -31,6 +33,11 @@ public class SignedIdentityDocumentEntity {
@JsonProperty("provider-service") public final String providerService;
@JsonProperty("zts-endpoint") public final URI ztsEndpoint;
@JsonProperty("document-version") public final int documentVersion;
+ @JsonProperty("configserver-hostname") public final String configServerHostname;
+ @JsonProperty("instance-hostname") public final String instanceHostname;
+ @JsonProperty("created-at") public final Instant createdAt;
+ @JsonProperty("ip-addresses") public final Set<String> ipAddresses;
+ @JsonProperty("identity-type") public final String identityType;
@JsonCreator
public SignedIdentityDocumentEntity(@JsonProperty("identity-document") String rawIdentityDocument,
@@ -40,7 +47,12 @@ public class SignedIdentityDocumentEntity {
@JsonProperty("dns-suffix") String dnsSuffix,
@JsonProperty("provider-service") String providerService,
@JsonProperty("zts-endpoint") URI ztsEndpoint,
- @JsonProperty("document-version") int documentVersion) {
+ @JsonProperty("document-version") int documentVersion,
+ @JsonProperty("configserver-hostname") String configServerHostname,
+ @JsonProperty("instance-hostname") String instanceHostname,
+ @JsonProperty("created-at") Instant createdAt,
+ @JsonProperty("ip-addresses") Set<String> ipAddresses,
+ @JsonProperty("identity-type") String identityType) {
this.rawIdentityDocument = rawIdentityDocument;
this.identityDocument = parseIdentityDocument(rawIdentityDocument);
this.signature = signature;
@@ -50,6 +62,11 @@ public class SignedIdentityDocumentEntity {
this.providerService = providerService;
this.ztsEndpoint = ztsEndpoint;
this.documentVersion = documentVersion;
+ this.configServerHostname = configServerHostname;
+ this.instanceHostname = instanceHostname;
+ this.createdAt = createdAt;
+ this.ipAddresses = ipAddresses;
+ this.identityType = identityType;
}
private static IdentityDocumentEntity parseIdentityDocument(String rawIdentityDocument) {
@@ -73,7 +90,16 @@ public class SignedIdentityDocumentEntity {
", identityDocument=" + identityDocument +
", signature='" + signature + '\'' +
", signingKeyVersion=" + signingKeyVersion +
+ ", providerUniqueId='" + providerUniqueId + '\'' +
+ ", dnsSuffix='" + dnsSuffix + '\'' +
+ ", providerService='" + providerService + '\'' +
+ ", ztsEndpoint=" + ztsEndpoint +
", documentVersion=" + documentVersion +
+ ", configServerHostname='" + configServerHostname + '\'' +
+ ", instanceHostname='" + instanceHostname + '\'' +
+ ", createdAt=" + createdAt +
+ ", ipAddresses=" + ipAddresses +
+ ", identityType=" + identityType +
'}';
}
@@ -86,11 +112,20 @@ public class SignedIdentityDocumentEntity {
documentVersion == that.documentVersion &&
Objects.equals(rawIdentityDocument, that.rawIdentityDocument) &&
Objects.equals(identityDocument, that.identityDocument) &&
- Objects.equals(signature, that.signature);
+ Objects.equals(signature, that.signature) &&
+ Objects.equals(providerUniqueId, that.providerUniqueId) &&
+ Objects.equals(dnsSuffix, that.dnsSuffix) &&
+ Objects.equals(providerService, that.providerService) &&
+ Objects.equals(ztsEndpoint, that.ztsEndpoint) &&
+ Objects.equals(configServerHostname, that.configServerHostname) &&
+ Objects.equals(instanceHostname, that.instanceHostname) &&
+ Objects.equals(createdAt, that.createdAt) &&
+ Objects.equals(ipAddresses, that.ipAddresses) &&
+ Objects.equals(identityType, identityType);
}
@Override
public int hashCode() {
- return Objects.hash(rawIdentityDocument, identityDocument, signature, signingKeyVersion, documentVersion);
+ return Objects.hash(rawIdentityDocument, identityDocument, signature, signingKeyVersion, providerUniqueId, dnsSuffix, providerService, ztsEndpoint, documentVersion, configServerHostname, instanceHostname, createdAt, ipAddresses, identityType);
}
}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java
index 3127752ac7d..103c087638d 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java
@@ -1,6 +1,7 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.athenz.identityprovider.api.bindings;
+import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Objects;
@@ -24,14 +25,18 @@ public class VespaUniqueInstanceIdEntity {
public final String clusterId;
@JsonProperty("cluster-index")
public final int clusterIndex;
+ @JsonProperty("type")
+ public final String type;
+ @JsonCreator
public VespaUniqueInstanceIdEntity(@JsonProperty("tenant") String tenant,
@JsonProperty("application") String application,
@JsonProperty("environment") String environment,
@JsonProperty("region") String region,
@JsonProperty("instance") String instance,
@JsonProperty("cluster-id") String clusterId,
- @JsonProperty("cluster-index") int clusterIndex) {
+ @JsonProperty("cluster-index") int clusterIndex,
+ @JsonProperty("type") String type) {
this.tenant = tenant;
this.application = application;
this.environment = environment;
@@ -39,8 +44,21 @@ public class VespaUniqueInstanceIdEntity {
this.instance = instance;
this.clusterId = clusterId;
this.clusterIndex = clusterIndex;
+ this.type = type;
}
+ @Deprecated
+ public VespaUniqueInstanceIdEntity(String tenant,
+ String application,
+ String environment,
+ String region,
+ String instance,
+ String clusterId,
+ int clusterIndex) {
+ this(tenant, application, environment, region, instance, clusterId, clusterIndex, null);
+ }
+
+
@Override
public String toString() {
return "VespaUniqueInstanceIdEntity{" +
@@ -51,6 +69,7 @@ public class VespaUniqueInstanceIdEntity {
", instance='" + instance + '\'' +
", clusterId='" + clusterId + '\'' +
", clusterIndex=" + clusterIndex +
+ ", type='" + type + '\'' +
'}';
}
@@ -65,11 +84,12 @@ public class VespaUniqueInstanceIdEntity {
Objects.equals(environment, that.environment) &&
Objects.equals(region, that.region) &&
Objects.equals(instance, that.instance) &&
- Objects.equals(clusterId, that.clusterId);
+ Objects.equals(clusterId, that.clusterId) &&
+ Objects.equals(type, that.type);
}
@Override
public int hashCode() {
- return Objects.hash(tenant, application, environment, region, instance, clusterId, clusterIndex);
+ return Objects.hash(tenant, application, environment, region, instance, clusterId, clusterIndex, type);
}
}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java
index 96e93ca419d..e8ef2d9f97e 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.athenz.identityprovider.client;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.yahoo.container.core.identity.IdentityConfig;
import com.yahoo.vespa.athenz.api.AthenzService;
import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
@@ -28,7 +29,7 @@ import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS;
*/
class AthenzCredentialsService {
- private static final ObjectMapper mapper = new ObjectMapper();
+ private static final ObjectMapper mapper = new ObjectMapper().registerModule(new JavaTimeModule());
private final IdentityConfig identityConfig;
private final IdentityDocumentClient identityDocumentClient;
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java
index 90d1312c9f9..f92956f7961 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java
@@ -2,14 +2,11 @@
package com.yahoo.vespa.athenz.identityprovider.client;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.yahoo.vespa.athenz.api.AthenzService;
import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient;
import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
-import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId;
import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity;
-import com.yahoo.vespa.athenz.utils.AthenzIdentities;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
@@ -82,15 +79,7 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient {
String responseContent = EntityUtils.toString(response.getEntity());
if (HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) {
SignedIdentityDocumentEntity entity = objectMapper.readValue(responseContent, SignedIdentityDocumentEntity.class);
- return new SignedIdentityDocument(
- EntityBindingsMapper.toIdentityDocument(entity.identityDocument),
- entity.signature,
- entity.signingKeyVersion,
- VespaUniqueInstanceId.fromDottedString(entity.providerUniqueId),
- entity.dnsSuffix,
- (AthenzService) AthenzIdentities.from(entity.providerService),
- entity.ztsEndpoint,
- entity.documentVersion);
+ return EntityBindingsMapper.toSignedIdentityDocument(entity);
} else {
throw new RuntimeException(
String.format(
diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java
index 8c4e4c1262d..86b6c566987 100644
--- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java
+++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java
@@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.identityprovider.api;
import org.junit.Test;
+import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.*;
import static org.junit.Assert.*;
/**
@@ -12,6 +13,18 @@ public class VespaUniqueInstanceIdTest {
@Test
public void can_serialize_to_and_deserialize_from_string() {
VespaUniqueInstanceId id =
+ new VespaUniqueInstanceId(1, "cluster-id", "instance", "application", "tenant", "region", "environment", TENANT);
+ String stringRepresentation = id.asDottedString();
+ String expectedStringRepresentation = "1.cluster-id.instance.application.tenant.region.environment.tenant";
+ assertEquals(expectedStringRepresentation, stringRepresentation);
+ VespaUniqueInstanceId deserializedId = VespaUniqueInstanceId.fromDottedString(stringRepresentation);
+ assertEquals(id, deserializedId);
+ }
+
+ // TODO Remove support for legacy representation without type
+ @Test
+ public void supports_legacy_representation_without_type() {
+ VespaUniqueInstanceId id =
new VespaUniqueInstanceId(1, "cluster-id", "instance", "application", "tenant", "region", "environment");
String stringRepresentation = id.asDottedString();
String expectedStringRepresentation = "1.cluster-id.instance.application.tenant.region.environment";
diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java
index 2e9b29f5327..7ad465a7d80 100644
--- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java
+++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java
@@ -11,6 +11,7 @@ import com.yahoo.test.ManualClock;
import com.yahoo.vespa.athenz.api.AthenzService;
import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument;
+import com.yahoo.vespa.athenz.identityprovider.api.IdentityType;
import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId;
import com.yahoo.vespa.athenz.tls.KeyStoreBuilder;
@@ -132,7 +133,7 @@ public class AthenzIdentityProviderImplTest {
}
private static String getIdentityDocument() throws JsonProcessingException {
- VespaUniqueInstanceId instanceId = new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", "us-north-1", "dev");
+ VespaUniqueInstanceId instanceId = new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", "us-north-1", "dev", IdentityType.TENANT);
SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument(
new IdentityDocument(instanceId, "localhost", "x.y.com", Instant.EPOCH, Collections.emptySet()),
"dummysignature",
@@ -141,7 +142,12 @@ public class AthenzIdentityProviderImplTest {
"dev-us-north-1.vespa.cloud",
new AthenzService("vespa.vespa.provider_dev_us-north-1"),
URI.create("https://zts:4443/zts/v1"),
- 1);
+ 1,
+ "localhost",
+ "x.y.com",
+ Instant.EPOCH,
+ Collections.emptySet(),
+ IdentityType.TENANT);
return new ObjectMapper().registerModule(new JavaTimeModule())
.writeValueAsString(EntityBindingsMapper.toSignedIdentityDocumentEntity(signedIdentityDocument));
diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp
index 0225b788e68..ea9684efe01 100644
--- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp
+++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp
@@ -72,7 +72,7 @@ nbostream::operator = (const nbostream & rhs) {
return *this;
}
-nbostream::~nbostream() { }
+nbostream::~nbostream() = default;
void nbostream::fail(State s)
{
diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp
index dff744a0a41..01d0f91dfa3 100644
--- a/vespalib/src/vespa/vespalib/util/alloc.cpp
+++ b/vespalib/src/vespa/vespalib/util/alloc.cpp
@@ -186,6 +186,7 @@ struct MMapLimitAndAlignmentHash {
};
using AutoAllocatorsMap = std::unordered_map<MMapLimitAndAlignment, AutoAllocator::UP, MMapLimitAndAlignmentHash>;
+using AutoAllocatorsMapWithDefault = std::pair<AutoAllocatorsMap, alloc::MemoryAllocator *>;
void createAlignedAutoAllocators(AutoAllocatorsMap & map, size_t mmapLimit) {
for (size_t alignment : {0,0x200, 0x400, 0x1000}) {
@@ -197,7 +198,8 @@ void createAlignedAutoAllocators(AutoAllocatorsMap & map, size_t mmapLimit) {
}
}
-AutoAllocatorsMap createAutoAllocators() {
+AutoAllocatorsMap
+createAutoAllocators() {
AutoAllocatorsMap map;
map.reserve(3*5);
for (size_t pages : {1,2,4,8,16}) {
@@ -207,7 +209,30 @@ AutoAllocatorsMap createAutoAllocators() {
return map;
}
-AutoAllocatorsMap _G_availableAutoAllocators = createAutoAllocators();
+MemoryAllocator &
+getAutoAllocator(AutoAllocatorsMap & map, size_t mmapLimit, size_t alignment) {
+ MMapLimitAndAlignment key(mmapLimit, alignment);
+ auto found = map.find(key);
+ if (found == map.end()) {
+ throw IllegalArgumentException(make_string("We currently have no support for mmapLimit(%0lx) and alignment(%0lx)", mmapLimit, alignment));
+ }
+ return *(found->second);
+}
+
+MemoryAllocator &
+getDefaultAutoAllocator(AutoAllocatorsMap & map) {
+ return getAutoAllocator(map, 1 * MemoryAllocator::HUGEPAGE_SIZE, 0);
+}
+
+AutoAllocatorsMapWithDefault
+createAutoAllocatorsWithDefault() {
+ AutoAllocatorsMapWithDefault tmp(createAutoAllocators(), nullptr);
+ tmp.second = &getDefaultAutoAllocator(tmp.first);
+ return tmp;
+}
+
+
+AutoAllocatorsMapWithDefault _G_availableAutoAllocators = createAutoAllocatorsWithDefault();
alloc::HeapAllocator _G_heapAllocatorDefault;
alloc::AlignedHeapAllocator _G_4KalignedHeapAllocator(1024);
alloc::AlignedHeapAllocator _G_1KalignedHeapAllocator(4096);
@@ -216,11 +241,13 @@ alloc::MMapAllocator _G_mmapAllocatorDefault;
}
-MemoryAllocator & HeapAllocator::getDefault() {
+MemoryAllocator &
+HeapAllocator::getDefault() {
return _G_heapAllocatorDefault;
}
-MemoryAllocator & AlignedHeapAllocator::get4K() {
+MemoryAllocator &
+AlignedHeapAllocator::get4K() {
return _G_4KalignedHeapAllocator;
}
@@ -228,25 +255,23 @@ MemoryAllocator & AlignedHeapAllocator::get1K() {
return _G_1KalignedHeapAllocator;
}
-MemoryAllocator & AlignedHeapAllocator::get512B() {
+MemoryAllocator &
+AlignedHeapAllocator::get512B() {
return _G_512BalignedHeapAllocator;
}
-MemoryAllocator & MMapAllocator::getDefault() {
+MemoryAllocator &
+MMapAllocator::getDefault() {
return _G_mmapAllocatorDefault;
}
-MemoryAllocator & AutoAllocator::getDefault() {
- return getAllocator(1 * MemoryAllocator::HUGEPAGE_SIZE, 0);
+MemoryAllocator &
+AutoAllocator::getDefault() {
+ return *_G_availableAutoAllocators.second;
}
MemoryAllocator & AutoAllocator::getAllocator(size_t mmapLimit, size_t alignment) {
- MMapLimitAndAlignment key(mmapLimit, alignment);
- auto found = _G_availableAutoAllocators.find(key);
- if (found == _G_availableAutoAllocators.end()) {
- throw IllegalArgumentException(make_string("We currently have no support for mmapLimit(%0lx) and alignment(%0lx)", mmapLimit, alignment));
- }
- return *(found->second);
+ return getAutoAllocator(_G_availableAutoAllocators.first, mmapLimit, alignment);
}
MemoryAllocator::PtrAndSize
@@ -466,6 +491,12 @@ Alloc::allocMMap(size_t sz)
}
Alloc
+Alloc::alloc()
+{
+ return Alloc(&AutoAllocator::getDefault());
+}
+
+Alloc
Alloc::alloc(size_t sz, size_t mmapLimit, size_t alignment)
{
return Alloc(&AutoAllocator::getAllocator(mmapLimit, alignment), sz);
diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h
index 2c3de92c58e..b52cace45a5 100644
--- a/vespalib/src/vespa/vespalib/util/alloc.h
+++ b/vespalib/src/vespa/vespalib/util/alloc.h
@@ -88,7 +88,7 @@ public:
std::swap(_allocator, rhs._allocator);
}
Alloc create(size_t sz) const {
- return Alloc(_allocator, sz);
+ return (sz == 0) ? Alloc(_allocator) : Alloc(_allocator, sz);
}
static Alloc allocAlignedHeap(size_t sz, size_t alignment);
@@ -98,9 +98,11 @@ public:
* Optional alignment is assumed to be <= system page size, since mmap
* is always used when size is above limit.
*/
- static Alloc alloc(size_t sz=0, size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0);
+ static Alloc alloc(size_t sz, size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0);
+ static Alloc alloc();
private:
Alloc(const MemoryAllocator * allocator, size_t sz) : _alloc(allocator->alloc(sz)), _allocator(allocator) { }
+ Alloc(const MemoryAllocator * allocator) : _alloc(nullptr, 0), _allocator(allocator) { }
void clear() {
_alloc.first = nullptr;
_alloc.second = 0;