diff options
192 files changed, 7280 insertions, 4255 deletions
diff --git a/application/src/main/java/com/yahoo/application/container/JDisc.java b/application/src/main/java/com/yahoo/application/container/JDisc.java index ed0c29a3917..5554ae6a159 100644 --- a/application/src/main/java/com/yahoo/application/container/JDisc.java +++ b/application/src/main/java/com/yahoo/application/container/JDisc.java @@ -15,7 +15,6 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.container.Container; import com.yahoo.container.standalone.StandaloneContainerApplication; -import com.yahoo.container.standalone.StandaloneContainerApplication$; import com.yahoo.docproc.jdisc.DocumentProcessingHandler; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.handler.RequestHandler; @@ -58,10 +57,10 @@ public final class JDisc implements AutoCloseable { return new AbstractModule() { @Override protected void configure() { - bind(Path.class).annotatedWith(StandaloneContainerApplication.applicationPathName()).toInstance(path); - bind(ConfigModelRepo.class).annotatedWith(StandaloneContainerApplication.configModelRepoName()).toInstance(configModelRepo); + bind(Path.class).annotatedWith(StandaloneContainerApplication.APPLICATION_PATH_NAME).toInstance(path); + bind(ConfigModelRepo.class).annotatedWith(StandaloneContainerApplication.CONFIG_MODEL_REPO_NAME).toInstance(configModelRepo); bind(Boolean.class).annotatedWith( // below is an ugly hack to access fields from a scala object. - Names.named(StandaloneContainerApplication$.MODULE$.disableNetworkingAnnotation())).toInstance( + Names.named(StandaloneContainerApplication.DISABLE_NETWORKING_ANNOTATION)).toInstance( networking == Networking.disable); } }; diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 86d4defa861..982cb89f2bf 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -131,6 +131,14 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-deprecation</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> </plugin> </plugins> </build> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java index 728406c297f..59126fd023f 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java @@ -7,6 +7,7 @@ import com.yahoo.net.HostName; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; @@ -27,7 +28,10 @@ import java.util.Objects; import java.util.Set; /** + * Generates a signed identity document for a given hostname and type + * * @author mortent + * @author bjorncs */ public class IdentityDocumentGenerator { @@ -47,10 +51,10 @@ public class IdentityDocumentGenerator { this.keyProvider = keyProvider; } - public SignedIdentityDocument generateSignedIdentityDocument(String hostname) { + public SignedIdentityDocument generateSignedIdentityDocument(String hostname, IdentityType identityType) { Node node = nodeRepository.getNode(hostname).orElseThrow(() -> new RuntimeException("Unable to find node " + hostname)); try { - IdentityDocument identityDocument = generateIdDocument(node); + IdentityDocument identityDocument = generateIdDocument(node, identityType); String identityDocumentString = Utils.getMapper().writeValueAsString(EntityBindingsMapper.toIdentityDocumentEntity(identityDocument)); String encodedIdentityDocument = @@ -70,13 +74,18 @@ public class IdentityDocumentGenerator { toZoneDnsSuffix(zone, zoneConfig.certDnsSuffix()), new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()), URI.create(zoneConfig.ztsUrl()), - SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION); + SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION, + identityDocument.configServerHostname(), + identityDocument.instanceHostname(), + identityDocument.createdAt(), + identityDocument.ipAddresses(), + identityType); } catch (Exception e) { throw new RuntimeException("Exception generating identity document: " + e.getMessage(), e); } } - private IdentityDocument generateIdDocument(Node node) { + private IdentityDocument generateIdDocument(Node node, IdentityType identityType) { Allocation allocation = node.allocation().orElseThrow(() -> new RuntimeException("No allocation for node " + node.hostname())); VespaUniqueInstanceId providerUniqueId = new VespaUniqueInstanceId( allocation.membership().index(), @@ -85,17 +94,10 @@ public class IdentityDocumentGenerator { allocation.owner().application().value(), allocation.owner().tenant().value(), zone.region().value(), - zone.environment().value()); + zone.environment().value(), + identityType); - // TODO: Hack to allow access from docker containers to non-ipv6 services. - // Remove when yca-bridge is no longer needed Set<String> ips = new HashSet<>(node.ipAddresses()); - if(node.parentHostname().isPresent()) { - String parentHostName = node.parentHostname().get(); - nodeRepository.getNode(parentHostName) - .map(Node::ipAddresses) - .ifPresent(ips::addAll); - } return new IdentityDocument( providerUniqueId, HostName.getLocalhost(), diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java index 93668006e26..219e12c7223 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java @@ -6,6 +6,7 @@ import com.yahoo.container.jaxrs.annotation.Component; import com.yahoo.jdisc.http.servlet.ServletRequest; import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentApi; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodePrincipal; @@ -18,7 +19,6 @@ import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.util.logging.Logger; @@ -41,15 +41,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi { this.request = request; } - /** - * @deprecated Use {@link #getNodeIdentityDocument(String)} and {@link #getTenantIdentityDocument(String)} instead. - */ - @GET - @Produces(MediaType.APPLICATION_JSON) - @Deprecated - @Override - // TODO Make this method private when the rest api is not longer in use - public SignedIdentityDocumentEntity getIdentityDocument(@QueryParam("hostname") String hostname) { + private SignedIdentityDocumentEntity getIdentityDocument(String hostname, IdentityType identityType) { if (hostname == null) { throw new BadRequestException("The 'hostname' query parameter is missing"); } @@ -67,7 +59,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi { throw new ForbiddenException(); } try { - return EntityBindingsMapper.toSignedIdentityDocumentEntity(identityDocumentGenerator.generateSignedIdentityDocument(hostname)); + return EntityBindingsMapper.toSignedIdentityDocumentEntity(identityDocumentGenerator.generateSignedIdentityDocument(hostname, identityType)); } catch (Exception e) { String message = String.format("Unable to generate identity doument for '%s': %s", hostname, e.getMessage()); log.log(LogLevel.ERROR, message, e); @@ -80,7 +72,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi { @Path("/node/{host}") @Override public SignedIdentityDocumentEntity getNodeIdentityDocument(@PathParam("host") String host) { - return getIdentityDocument(host); + return getIdentityDocument(host, IdentityType.NODE); } @GET @@ -88,7 +80,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi { @Path("/tenant/{host}") @Override public SignedIdentityDocumentEntity getTenantIdentityDocument(@PathParam("host") String host) { - return getIdentityDocument(host); + return getIdentityDocument(host, IdentityType.TENANT); } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index e457df37946..0201c46b253 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -82,6 +82,7 @@ public class InstanceValidator { } // If/when we dont care about logging exactly whats wrong, this can be simplified + // TODO Use identity type to determine if this check should be performed boolean isSameIdentityAsInServicesXml(ApplicationId applicationId, String domain, String service) { Optional<ApplicationInfo> applicationInfo = superModelProvider.getSuperModel().getApplicationInfo(applicationId); diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index d7b061ca2f1..078ef1b7e39 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -15,6 +15,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; @@ -81,7 +82,7 @@ public class IdentityDocumentGeneratorTest { AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix, ZONE); IdentityDocumentGenerator identityDocumentGenerator = new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider); - SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname); + SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname, IdentityType.TENANT); // Verify attributes assertEquals(containerHostname, signedIdentityDocument.identityDocument().instanceHostname()); @@ -92,11 +93,11 @@ public class IdentityDocumentGeneratorTest { assertEquals(expectedZoneDnsSuffix, signedIdentityDocument.dnsSuffix()); VespaUniqueInstanceId expectedProviderUniqueId = - new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", region, environment); + new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", region, environment, IdentityType.TENANT); assertEquals(expectedProviderUniqueId, signedIdentityDocument.providerUniqueId()); - // Validate that both parent and container ips are present - assertThat(signedIdentityDocument.identityDocument().ipAddresses(), Matchers.containsInAnyOrder("127.0.0.1", "::1")); + // Validate that container ips are present + assertThat(signedIdentityDocument.identityDocument().ipAddresses(), Matchers.containsInAnyOrder("::1")); SignedIdentityDocumentEntity signedIdentityDocumentEntity = EntityBindingsMapper.toSignedIdentityDocumentEntity(signedIdentityDocument); diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index 54786c86cd3..54411b424eb 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -143,7 +143,12 @@ public class InstanceValidatorTest { "dnssuffix", "service", URI.create("http://localhost/zts"), - 1)); + 1, + identityDocument.configServerHostname, + identityDocument.instanceHostname, + identityDocument.createdAt, + identityDocument.ipAddresses, + null)); // TODO Remove support for legacy representation without type } catch (Exception e) { throw new RuntimeException(e); } diff --git a/config-lib/src/main/java/com/yahoo/config/FileReference.java b/config-lib/src/main/java/com/yahoo/config/FileReference.java index 5f0bc275bad..7d455c58b30 100755 --- a/config-lib/src/main/java/com/yahoo/config/FileReference.java +++ b/config-lib/src/main/java/com/yahoo/config/FileReference.java @@ -7,19 +7,20 @@ import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * An immutable file reference that can only be created from classes within the same package. * This is to prevent clients from creating arbitrary and invalid file references. * - * @author tonytv + * @author Tony Vaagenes */ public final class FileReference { private final String value; public FileReference(String value) { - this.value = value; + this.value = Objects.requireNonNull(value); } public String value() { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java index ac1bcfa542a..9b457f49bd2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java @@ -23,12 +23,18 @@ public interface FileDistribution { */ void startDownload(String hostName, int port, Set<FileReference> fileReferences); + // TODO: Remove when 6.244 is oldest version in use + @Deprecated static String getDefaultFileDBRoot() { return Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"); } + // TODO: Remove when 6.244 is oldest version in use + @Deprecated static File getDefaultFileDBPath() { return new File(getDefaultFileDBRoot()); } + File getFileReferencesDir(); + } diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 1857cc33d64..3ef9925510c 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -14,30 +14,29 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-model-api</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> + <artifactId>fat-model-dependencies</artifactId> <version>${project.version}</version> - <scope>provided</scope> + <type>pom</type> </dependency> + <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-model</artifactId> - <version>${project.version}</version> + <!-- TODO: remove, we probably don't need version 13. --> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>13.0.1</version> </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-lib</artifactId> + <artifactId>config-model-api</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>provided-dependencies</artifactId> + <artifactId>config-provisioning</artifactId> <version>${project.version}</version> + <scope>provided</scope> <exclusions> <exclusion> <groupId>com.google.inject</groupId> @@ -45,6 +44,8 @@ </exclusion> </exclusions> </dependency> + + <!-- TODO: remove all test deps, should not be needed --> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> @@ -60,35 +61,6 @@ <artifactId>guava-testlib</artifactId> <version>17.0</version> <scope>test</scope> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>13.0.1</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>configdefinitions</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-application-package</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>configgen</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-bundle</artifactId> - <version>${project.version}</version> <exclusions> <exclusion> <groupId>com.yahoo.vespa</groupId> @@ -102,109 +74,9 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>simplemetrics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>metrics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-disc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>yolean</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> <version>${project.version}</version> <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>documentapi</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vdslib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>document</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>linguistics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>statistics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus-disc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-messagebus</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>processing</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>chain</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>docproc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-search</artifactId> - <version>${project.version}</version> <exclusions> <exclusion> <!-- OPTIMIZATION: very large (44 MB) and only used for query sorting --> @@ -213,50 +85,6 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-search-and-docproc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>logd</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchcore</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>storage</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vsm</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>indexinglanguage</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>searchsummary</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.major-version}</artifactId> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_http_service</artifactId> - <version>${project.version}</version> - </dependency> </dependencies> <build> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index c7ca1a33ff2..fc75f7a19fc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -5,6 +5,7 @@ import com.yahoo.config.subscription.ConfigInstanceUtil; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.Search; @@ -44,39 +45,59 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives everything from a field */ @Override protected void derive(ImmutableSDField field, Search search) { - boolean fieldIsArrayOfSimpleStruct = isArrayOfSimpleStruct(field); - if (field.usesStructOrMap() && - !fieldIsArrayOfSimpleStruct && - !field.getDataType().equals(PositionDataType.INSTANCE) && - !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { - return; // Ignore struct fields for indexed search (only implemented for streaming search) + if (unsupportedFieldType(field)) { + return; // Ignore majority of struct fields for indexed search (only implemented for streaming search) } if (field.isImportedField()) { deriveImportedAttributes(field); - } else if (fieldIsArrayOfSimpleStruct) { + } else if (isArrayOfSimpleStruct(field)) { deriveArrayOfSimpleStruct(field); + } else if (isMapOfSimpleStruct(field)) { + deriveMapOfSimpleStruct(field); } else { deriveAttributes(field); } } + private static boolean unsupportedFieldType(ImmutableSDField field) { + return (field.usesStructOrMap() && + !isArrayOfSimpleStruct(field) && + !isMapOfSimpleStruct(field) && + !field.getDataType().equals(PositionDataType.INSTANCE) && + !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); + } + private static boolean isArrayOfSimpleStruct(ImmutableSDField field) { DataType fieldType = field.getDataType(); if (fieldType instanceof ArrayDataType) { ArrayDataType arrayType = (ArrayDataType)fieldType; - DataType nestedType = arrayType.getNestedType(); - if (nestedType instanceof StructDataType && - !(nestedType.equals(PositionDataType.INSTANCE))) { - StructDataType structType = (StructDataType)nestedType; - for (Field innerField : structType.getFields()) { - if (!isPrimitiveType(innerField.getDataType())) { - return false; - } + return isSimpleStruct(arrayType.getNestedType()); + } else { + return false; + } + } + + private static boolean isMapOfSimpleStruct(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof MapDataType) { + MapDataType mapType = (MapDataType)fieldType; + return isPrimitiveType(mapType.getKeyType()) && + isSimpleStruct(mapType.getValueType()); + } else { + return false; + } + } + + private static boolean isSimpleStruct(DataType type) { + if (type instanceof StructDataType && + !(type.equals(PositionDataType.INSTANCE))) { + StructDataType structType = (StructDataType) type; + for (Field innerField : structType.getFields()) { + if (!isPrimitiveType(innerField.getDataType())) { + return false; } - return true; - } else { - return false; } + return true; } else { return false; } @@ -138,14 +159,33 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce private void deriveArrayOfSimpleStruct(ImmutableSDField field) { for (ImmutableSDField structField : field.getStructFields()) { - for (Attribute attribute : structField.getAttributes().values()) { - if (structField.getName().equals(attribute.getName())) { - attributes.put(attribute.getName(), attribute.convertToArray()); - } + deriveAttributesAsArrayType(structField); + } + } + + private void deriveAttributesAsArrayType(ImmutableSDField field) { + for (Attribute attribute : field.getAttributes().values()) { + if (field.getName().equals(attribute.getName())) { + attributes.put(attribute.getName(), attribute.convertToArray()); } } } + private void deriveMapOfSimpleStruct(ImmutableSDField field) { + deriveMapKeyField(field.getStructField("key")); + deriveMapValueField(field.getStructField("value")); + } + + private void deriveMapKeyField(ImmutableSDField keyField) { + deriveAttributesAsArrayType(keyField); + } + + private void deriveMapValueField(ImmutableSDField valueField) { + for (ImmutableSDField structField : valueField.getStructFields()) { + deriveAttributesAsArrayType(structField); + } + } + /** Returns a read only attribute iterator */ public Iterator attributeIterator() { return attributes().iterator(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java index 4bdef0607a2..0661ef67131 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/GlobalDistributionValidator.java @@ -21,45 +21,11 @@ import static java.util.stream.Collectors.toSet; public class GlobalDistributionValidator { public void validate(Map<String, NewDocumentType> documentDefinitions, - Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy, - boolean enableMultipleBucketSpaces) { - if (!enableMultipleBucketSpaces) { - verifyGlobalDocumentsHaveRequiredRedundancy(globallyDistributedDocuments, redundancy); - verifySearchableCopiesIsSameAsRedundancy(globallyDistributedDocuments, redundancy); - } + Set<NewDocumentType> globallyDistributedDocuments) { verifyReferredDocumentsArePresent(documentDefinitions); verifyReferredDocumentsAreGlobal(documentDefinitions, globallyDistributedDocuments); } - private static void verifyGlobalDocumentsHaveRequiredRedundancy(Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy) { - if (!globallyDistributedDocuments.isEmpty() && !redundancy.isEffectivelyGloballyDistributed()) { - throw new IllegalArgumentException( - String.format( - "The following document types are marked as global, " + - "but do not have high enough redundancy to make the documents globally distributed: %s. " + - "Redundancy is %d, expected %d.", - asPrintableString(toDocumentNameStream(globallyDistributedDocuments)), - redundancy.effectiveFinalRedundancy(), - redundancy.totalNodes())); - } - } - - private static void verifySearchableCopiesIsSameAsRedundancy(Set<NewDocumentType> globallyDistributedDocuments, - Redundancy redundancy) { - if (!globallyDistributedDocuments.isEmpty() && - redundancy.effectiveReadyCopies() != redundancy.effectiveFinalRedundancy()) { - throw new IllegalArgumentException( - String.format( - "The following document types have the number of searchable copies less than redundancy: %s. " + - "Searchable copies is %d, while redundancy is %d.", - asPrintableString(toDocumentNameStream(globallyDistributedDocuments)), - redundancy.effectiveReadyCopies(), - redundancy.effectiveFinalRedundancy())); - } - } - private static void verifyReferredDocumentsArePresent(Map<String, NewDocumentType> documentDefinitions) { Set<NewDocumentType.Name> unknowDocuments = getReferencedDocuments(documentDefinitions) .filter(name -> !documentDefinitions.containsKey(name.toString())) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 0119bced095..68ae4d2b242 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -61,14 +61,12 @@ public class ContentCluster extends AbstractConfigProducer implements MessagetyperouteselectorpolicyConfig.Producer, BucketspacesConfig.Producer { - // TODO: Make private private String documentSelection; private ContentSearchCluster search; private final boolean isHostedVespa; private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; - // Experimental flag (TODO: remove when feature is enabled by default) - private boolean enableMultipleBucketSpaces = false; + private boolean forceEnableMultipleBucketSpaces = false; private com.yahoo.vespa.model.content.StorageGroup rootGroup; private StorageCluster storageNodes; private DistributorCluster distributorNodes; @@ -250,7 +248,7 @@ public class ContentCluster extends AbstractConfigProducer implements private void setupExperimental(ContentCluster cluster, ModelElement experimental) { Boolean enableMultipleBucketSpaces = experimental.childAsBoolean("enable-multiple-bucket-spaces"); if (enableMultipleBucketSpaces != null) { - cluster.enableMultipleBucketSpaces = enableMultipleBucketSpaces; + cluster.forceEnableMultipleBucketSpaces = enableMultipleBucketSpaces; } } @@ -596,13 +594,13 @@ public class ContentCluster extends AbstractConfigProducer implements builder.min_distributor_up_ratio(0); builder.min_storage_up_ratio(0); } - builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces); + builder.enable_multiple_bucket_spaces(true); // Telling the controller whether we actually _have_ global document types lets // it selectively enable or disable constraints that aren't needed when these // are not are present, even if full protocol and backend support is enabled // for multiple bucket spaces. Basically, if you don't use it, you don't // pay for it. - builder.cluster_has_global_document_types(enableMultipleBucketSpaces && !globallyDistributedDocuments.isEmpty()); + builder.cluster_has_global_document_types(!globallyDistributedDocuments.isEmpty()); } @Override @@ -646,7 +644,7 @@ public class ContentCluster extends AbstractConfigProducer implements } } new ReservedDocumentTypeNameValidator().validate(documentDefinitions); - new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments, redundancy, enableMultipleBucketSpaces); + new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments); } public static Map<String, Integer> METRIC_INDEX_MAP = new TreeMap<>(); @@ -727,11 +725,13 @@ public class ContentCluster extends AbstractConfigProducer implements for (NewDocumentType docType : getDocumentDefinitions().values()) { BucketspacesConfig.Documenttype.Builder docTypeBuilder = new BucketspacesConfig.Documenttype.Builder(); docTypeBuilder.name(docType.getName()); - String bucketSpace = ((enableMultipleBucketSpaces && isGloballyDistributed(docType)) - ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); + String bucketSpace = (isGloballyDistributed(docType) ? GLOBAL_BUCKET_SPACE : DEFAULT_BUCKET_SPACE); docTypeBuilder.bucketspace(bucketSpace); builder.documenttype(docTypeBuilder); } - builder.enable_multiple_bucket_spaces(enableMultipleBucketSpaces); + // NOTE: this config is kept around to allow the use of multiple bucket spaces + // on older versions of Vespa. It is for all intents and purposes a no-op in + // newer versions where multiple bucket spaces are enabled by default. + builder.enable_multiple_bucket_spaces(forceEnableMultipleBucketSpaces); } } diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg new file mode 100644 index 00000000000..604fc1f6ea7 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -0,0 +1,60 @@ +attribute[0].name "elem_map.key" +attribute[0].datatype STRING +attribute[0].collectiontype ARRAY +attribute[0].removeifzero false +attribute[0].createifnonexistent false +attribute[0].fastsearch false +attribute[0].huge false +attribute[0].sortascending true +attribute[0].sortfunction UCA +attribute[0].sortstrength PRIMARY +attribute[0].sortlocale "" +attribute[0].enablebitvectors false +attribute[0].enableonlybitvector false +attribute[0].fastaccess false +attribute[0].arity 8 +attribute[0].lowerbound -9223372036854775808 +attribute[0].upperbound 9223372036854775807 +attribute[0].densepostinglistthreshold 0.4 +attribute[0].tensortype "" +attribute[0].imported false +attribute[1].name "elem_map.value.name" +attribute[1].datatype STRING +attribute[1].collectiontype ARRAY +attribute[1].removeifzero false +attribute[1].createifnonexistent false +attribute[1].fastsearch false +attribute[1].huge false +attribute[1].sortascending true +attribute[1].sortfunction UCA +attribute[1].sortstrength PRIMARY +attribute[1].sortlocale "" +attribute[1].enablebitvectors false +attribute[1].enableonlybitvector false +attribute[1].fastaccess false +attribute[1].arity 8 +attribute[1].lowerbound -9223372036854775808 +attribute[1].upperbound 9223372036854775807 +attribute[1].densepostinglistthreshold 0.4 +attribute[1].tensortype "" +attribute[1].imported false +attribute[2].name "elem_map.value.weight" +attribute[2].datatype INT32 +attribute[2].collectiontype ARRAY +attribute[2].removeifzero false +attribute[2].createifnonexistent false +attribute[2].fastsearch false +attribute[2].huge false +attribute[2].sortascending true +attribute[2].sortfunction UCA +attribute[2].sortstrength PRIMARY +attribute[2].sortlocale "" +attribute[2].enablebitvectors false +attribute[2].enableonlybitvector false +attribute[2].fastaccess false +attribute[2].arity 8 +attribute[2].lowerbound -9223372036854775808 +attribute[2].upperbound 9223372036854775807 +attribute[2].densepostinglistthreshold 0.4 +attribute[2].tensortype "" +attribute[2].imported false
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg new file mode 100644 index 00000000000..7af49d95d09 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -0,0 +1,11 @@ +defaultsummaryid 653486243 +classes[0].id 653486243 +classes[0].name "default" +classes[0].fields[0].name "elem_map" +classes[0].fields[0].type "jsonstring" +classes[0].fields[1].name "rankfeatures" +classes[0].fields[1].type "featuredata" +classes[0].fields[2].name "summaryfeatures" +classes[0].fields[2].type "featuredata" +classes[0].fields[3].name "documentid" +classes[0].fields[3].type "longstring"
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg new file mode 100644 index 00000000000..42b6e811ee6 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg @@ -0,0 +1,7 @@ +defaultoutputclass -1 +override[0].field "rankfeatures" +override[0].command "rankfeatures" +override[0].arguments "" +override[1].field "summaryfeatures" +override[1].command "summaryfeatures" +override[1].arguments ""
\ No newline at end of file diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd new file mode 100644 index 00000000000..cb2eac4ed78 --- /dev/null +++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd @@ -0,0 +1,20 @@ +search test { + document test { + struct elem { + field name type string {} + field weight type int {} + } + field elem_map type map<string,elem> { + indexing: summary + struct-field key { + indexing: attribute + } + struct-field value.name { + indexing: attribute + } + struct-field value.weight { + indexing: attribute + } + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 990ebe7f993..c3cfcae66e6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -79,6 +79,17 @@ public class AttributeListTestCase extends SearchDefinitionTestCase { assertTrue(!attributes.hasNext()); } + @Test + public void map_of_struct_field_is_derived_into_array_attributes() throws IOException, ParseException { + Search search = SearchBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Iterator<Attribute> attributes = new AttributeFields(search).attributeIterator(); + + assertAttribute("elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, attributes.next()); + assertAttribute("elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, attributes.next()); + assertTrue(!attributes.hasNext()); + } + private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, Attribute attr) { assertEquals(name, attr.getName()); assertEquals(type, attr.getType()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java index 9e73edf9b35..72c7aab4a39 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java @@ -18,4 +18,14 @@ public class AttributesTestCase extends AbstractExportingTestCase { assertCorrectDeriving("attributes"); } + @Test + public void testArrayOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("array_of_struct_attribute"); + } + + @Test + public void testMapOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("map_of_struct_attribute"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index dc2d3b7cea1..4600f6ae4c6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -139,9 +139,4 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectDeriving("tensor"); } - @Test - public void testArrayOfStructAttribute() throws IOException, ParseException { - assertCorrectDeriving("array_of_struct_attribute"); - } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 98177b4ada0..0156128f7ca 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -173,17 +173,17 @@ public class ContentSearchClusterTest { } @Test - public void require_that_all_document_types_belong_to_default_bucket_space_by_default() throws Exception { + public void require_that_document_types_belong_to_correct_bucket_spaces() throws Exception { BucketspacesConfig config = getBucketspacesConfig(createClusterWithGlobalType()); assertEquals(2, config.documenttype().size()); - assertDocumentType("global", "default", config.documenttype(0)); + assertDocumentType("global", "global", config.documenttype(0)); assertDocumentType("regular", "default", config.documenttype(1)); // Safeguard against flipping the switch assertFalse(config.enable_multiple_bucket_spaces()); } @Test - public void require_that_multiple_bucket_spaces_can_be_enabled() throws Exception { + public void require_that_multiple_bucket_spaces_can_be_force_enabled() throws Exception { ContentCluster cluster = createClusterWithMultipleBucketSpacesEnabled(); { BucketspacesConfig config = getBucketspacesConfig(cluster); @@ -210,9 +210,9 @@ public class ContentSearchClusterTest { } @Test - public void controller_global_documents_config_forced_to_false_if_multiple_spaces_not_enabled() throws Exception { + public void controller_global_documents_config_always_enabled_even_without_experimental_flag_set() throws Exception { ContentCluster cluster = createClusterWithGlobalDocsButNotMultipleSpacesEnabled(); - assertFalse(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); + assertTrue(getFleetcontrollerConfig(cluster).cluster_has_global_document_types()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java index b8252f2f081..6506f7a08a8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java @@ -26,64 +26,16 @@ public class GlobalDistributionValidatorTest { public final ExpectedException exceptionRule = ExpectedException.none(); @Test - public void throws_exception_if_redudancy_does_not_imply_global_distribution() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); - - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types are marked as global, " + - "but do not have high enough redundancy to make the documents globally distributed: " + - "'bar', 'foo'. Redundancy is 2, expected 3."); - validate(fixture, redundancy); - } - - @Test - public void validation_of_redundancy_is_deactivated_if_multiple_bucket_spaces_is_enabled() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithoutGlobalDistributionAndTooFewSearchableCopies(); - - validate(fixture, redundancy, true); - } - - @Test - public void throws_exception_if_searchable_copies_too_low() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")) - .addGlobalDocument(createDocumentType("bar")); - Redundancy redundancy = createRedundancyWithTooFewSearchableCopies(); - - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The following document types have the number of searchable copies less than redundancy: " + - "'bar', 'foo'. Searchable copies is 1, while redundancy is 2."); - validate(fixture, redundancy); - } - - @Test - public void validation_succeeds_when_globally_distributed_and_enough_searchable_copies() { - Fixture fixture = new Fixture() - .addGlobalDocument(createDocumentType("foo")); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); - validate(fixture, redundancy); - } - - @Test public void validation_succeeds_on_no_documents() { new GlobalDistributionValidator() - .validate(emptyMap(), emptySet(), createRedundancyWithoutGlobalDistribution(), false); + .validate(emptyMap(), emptySet()); } @Test public void validation_succeeds_on_no_global_documents() { Fixture fixture = new Fixture() .addNonGlobalDocument(createDocumentType("foo")); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -92,11 +44,10 @@ public class GlobalDistributionValidatorTest { Fixture fixture = new Fixture() .addNonGlobalDocument(parent) .addNonGlobalDocument(createDocumentType("child", parent)); - Redundancy redundancy = createRedundancyWithoutGlobalDistribution(); exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage( "The following document types are referenced from other documents, but are not globally distributed: 'parent'"); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -105,8 +56,7 @@ public class GlobalDistributionValidatorTest { Fixture fixture = new Fixture() .addGlobalDocument(parent) .addNonGlobalDocument(createDocumentType("child", parent)); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -115,11 +65,10 @@ public class GlobalDistributionValidatorTest { NewDocumentType child = createDocumentType("child", unknown); Fixture fixture = new Fixture() .addNonGlobalDocument(child); - Redundancy redundancy = createRedundancyWithGlobalDistribution(); exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage( "The following document types are referenced from other documents, but are not listed in services.xml: 'unknown'"); - validate(fixture, redundancy); + validate(fixture); } @Test @@ -130,42 +79,14 @@ public class GlobalDistributionValidatorTest { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/global_distribution_validation/").create(); } - private static Redundancy createRedundancyWithGlobalDistribution() { - Redundancy redundancy = new Redundancy(2, 2, 2); - redundancy.setTotalNodes(2); - return redundancy; - } - - private static Redundancy createRedundancyWithoutGlobalDistribution() { - Redundancy redundancy = new Redundancy(2, 2, 2); - redundancy.setTotalNodes(3); - return redundancy; - } - - private static Redundancy createRedundancyWithTooFewSearchableCopies() { - Redundancy redundancy = new Redundancy(2, 2, 1); - redundancy.setTotalNodes(2); - return redundancy; - } - - private static Redundancy createRedundancyWithoutGlobalDistributionAndTooFewSearchableCopies() { - Redundancy redundancy = new Redundancy(2, 2, 1); - redundancy.setTotalNodes(3); - return redundancy; - } - private static NewDocumentType createDocumentType(String name, NewDocumentType... references) { Set<NewDocumentType.Name> documentReferences = Stream.of(references).map(NewDocumentType::getFullName).collect(toSet()); return new NewDocumentType(new NewDocumentType.Name(name), documentReferences); } - private static void validate(Fixture fixture, Redundancy redundancy) { - validate(fixture, redundancy, false); - } - - private static void validate(Fixture fixture, Redundancy redundancy, boolean enableMultipleBucketSpaces) { + private static void validate(Fixture fixture) { new GlobalDistributionValidator() - .validate(fixture.getDocumentTypes(), fixture.getGloballyDistributedDocuments(), redundancy, enableMultipleBucketSpaces); + .validate(fixture.getDocumentTypes(), fixture.getGloballyDistributedDocuments()); } private static class Fixture { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java index eabd0e5a7e0..131a5344116 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java @@ -7,10 +7,10 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.test.MockHosts; import org.junit.Test; +import java.io.File; import java.util.Arrays; import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -48,5 +48,10 @@ public class FileDistributorTestCase { public void startDownload(String hostName, int port, Set<FileReference> fileReferences) { filesToDownloadCalled++; } + + @Override + public File getFileReferencesDir() { + return null; + } } } diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java new file mode 100644 index 00000000000..8c0e276b3d3 --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index 281db6fb43d..281db6fb43d 100644 --- a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 5e81526dc53..bf4c9599f4a 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -13,10 +13,13 @@ zookeeperserver[].port int default=2181 zookeeper.barrierTimeout long default=120 # in seconds zookeeperLocalhostAffinity bool default=true -# Misc +# Directories configModelPluginDir[] string configServerDBDir string default="var/db/vespa/config_server/serverdb/" configDefinitionsDir string default="share/vespa/configdefinitions/" +fileReferencesDir string default="var/db/vespa/filedistribution/" + +# Misc sessionLifetime long default=3600 # in seconds masterGeneration long default=0 multitenant bool default=false diff --git a/configgen/pom.xml b/configgen/pom.xml index e8de114f8d5..be9cd733601 100644 --- a/configgen/pom.xml +++ b/configgen/pom.xml @@ -19,14 +19,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_${scala.major-version}</artifactId> - </dependency> </dependencies> <build> <plugins> @@ -63,32 +55,6 @@ <updateReleaseInfo>true</updateReleaseInfo> </configuration> </plugin> - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <id>compile</id> - <goals> - <goal>compile</goal> - </goals> - <phase>compile</phase> - </execution> - <execution> - <id>test-compile</id> - <goals> - <goal>testCompile</goal> - </goals> - <phase>test-compile</phase> - </execution> - <execution> - <phase>process-resources</phase> - <goals> - <goal>compile</goal> - </goals> - </execution> - </executions> - </plugin> </plugins> </build> </project> diff --git a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java new file mode 100644 index 00000000000..bf3fc2902a1 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java @@ -0,0 +1,351 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import com.yahoo.config.codegen.LeafCNode.FileLeaf; +import com.yahoo.config.codegen.LeafCNode.PathLeaf; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static com.yahoo.config.codegen.ConfigGenerator.boxedDataType; +import static com.yahoo.config.codegen.ConfigGenerator.indentCode; +import static com.yahoo.config.codegen.ConfigGenerator.nodeClass; +import static com.yahoo.config.codegen.ConfigGenerator.userDataType; +import static com.yahoo.config.codegen.JavaClassBuilder.INDENTATION; +import static com.yahoo.config.codegen.JavaClassBuilder.createUniqueSymbol; +import static com.yahoo.config.codegen.ReservedWords.INTERNAL_PREFIX; +import static java.util.Arrays.stream; + +/** + * @author gjoranv + * @author ollivir + */ + +public class BuilderGenerator { + public static String getBuilder(InnerCNode node) { + return getDeclaration(node) + "\n" + // + indentCode(INDENTATION, getUninitializedScalars(node) + "\n\n" + // + stream(node.getChildren()).map(BuilderGenerator::getBuilderFieldDefinition).collect(Collectors.joining("\n")) + + "\n\n" + // + getBuilderConstructors(node, nodeClass(node)) + "\n\n" + // + getOverrideMethod(node) + "\n\n" + // + getBuilderSetters(node) + "\n" + // + getSpecialRootBuilderCode(node)) + + "}"; + } + + private static String getDeclaration(InnerCNode node) { + String getInterfaces = (node.getParent() == null) ? "implements ConfigInstance.Builder" : "implements ConfigBuilder"; + + return "public static class Builder " + getInterfaces + " {"; + } + + private static String getSpecialRootBuilderCode(InnerCNode node) { + return (node.getParent() == null) ? "\n" + getDispatchCode() + "\n" : ""; + } + + private static String getDispatchCode() { + // Use full path to @Override, as users are free to define an inner node called + // 'override'. (summarymap.def does) + // The generated inner 'Override' class would otherwise be mistaken for the + // annotation. + return "@java.lang.Override\n" + // + "public final boolean dispatchGetConfig(ConfigInstance.Producer producer) {\n" + // + " if (producer instanceof Producer) {\n" + // + " ((Producer)producer).getConfig(this);\n" + // + " return true;\n" + // + " }\n" + // + " return false;\n" + // + "}\n" + // + "\n" + // + "@java.lang.Override\n" + // + "public final String getDefMd5() { return CONFIG_DEF_MD5; }\n" + // + "@java.lang.Override\n" + // + "public final String getDefName() { return CONFIG_DEF_NAME; }\n" + // + "@java.lang.Override\n" + // + "public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }"; + } + + private static String getUninitializedScalars(InnerCNode node) { + List<String> scalarsWithoutDefault = new ArrayList<>(); + for (CNode child : node.getChildren()) { + if (child instanceof LeafCNode && (!child.isArray && !child.isMap && ((LeafCNode) child).getDefaultValue() == null)) { + scalarsWithoutDefault.add("\"" + child.getName() + "\""); + } + } + + String uninitializedList = (scalarsWithoutDefault.size() > 0) + ? "Arrays.asList(\n" + indentCode(INDENTATION, String.join(",\n", scalarsWithoutDefault) + "\n)") + : ""; + + return "private Set<String> " + INTERNAL_PREFIX + "uninitialized = new HashSet<String>(" + uninitializedList + ");"; + } + + private static String getBuilderFieldDefinition(CNode node) { + if (node.isArray) { + return String.format("public List<%s> %s = new ArrayList<>();", builderType(node), node.getName()); + } else if (node.isMap) { + return String.format("public Map<String, %s> %s = new LinkedHashMap<>();", builderType(node), node.getName()); + } else if (node instanceof InnerCNode) { + return String.format("public %s %s = new %s();", builderType(node), node.getName(), builderType(node)); + } else if (node instanceof LeafCNode) { + return String.format("private %s %s = null;", boxedBuilderType((LeafCNode) node), node.getName()); + } else { + throw new IllegalStateException("Cannot produce builder field definition for node"); // Should not happen + } + } + + private static String getBuilderSetters(CNode node) { + List<String> elem = new ArrayList<>(); + CNode[] children = node.getChildren(); + + for (CNode child : children) { + if (child instanceof InnerCNode && child.isArray) { + elem.add(BuilderSetters.innerArraySetters((InnerCNode) child)); + } else if (child instanceof InnerCNode && child.isMap) { + elem.add(BuilderSetters.innerMapSetters(child)); + } else if (child instanceof LeafCNode && child.isArray) { + elem.add(BuilderSetters.leafArraySetters((LeafCNode) child)); + } else if (child instanceof LeafCNode && child.isMap) { + elem.add(BuilderSetters.leafMapSetters(child)); + } else if (child instanceof InnerCNode) { + elem.add(BuilderSetters.structSetter((InnerCNode) child)); + } else if (child instanceof LeafCNode) { + elem.add(BuilderSetters.scalarSetters((LeafCNode) child)); + } + } + return String.join("\n\n", elem); + } + + private static class BuilderSetters { + private static String structSetter(InnerCNode n) { + return "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "builder) {\n" + // + " " + n.getName() + " = " + INTERNAL_PREFIX + "builder;\n" + // + " return this;\n" + // + "}"; + } + + private static String innerArraySetters(InnerCNode n) { + return "/**\n" + // + " * Add the given builder to this builder's list of " + nodeClass(n) + " builders\n" + // + " * @param " + INTERNAL_PREFIX + "builder a builder\n" + // + " * @return this builder\n" + // + " */\n" + // + "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "builder) {\n" + // + " " + n.getName() + ".add(" + INTERNAL_PREFIX + "builder);\n" + // + " return this;\n" + // + "}\n" + // + "\n" + // + "/**\n" + // + " * Set the given list as this builder's list of " + nodeClass(n) + " builders\n" + // + " * @param __builders a list of builders\n" + // + " * @return this builder\n" + // + " */\n" + // + "public Builder " + n.getName() + "(List<" + builderType(n) + "> __builders) {\n" + // + " " + n.getName() + " = __builders;\n" + // + " return this;\n" + // + "}"; + } + + private static String publicLeafNodeSetters(LeafCNode n) { + return "public Builder " + n.getName() + "(" + builderType(n) + " " + INTERNAL_PREFIX + "value) {\n" + // + " " + n.getName() + ".add(" + INTERNAL_PREFIX + "value);\n" + // + " return this;\n" + // + "}\n" + // + "\n" + // + "public Builder " + n.getName() + "(Collection<" + builderType(n) + "> " + INTERNAL_PREFIX + "values) {\n" + // + " " + n.getName() + ".addAll(" + INTERNAL_PREFIX + "values);\n" + // + " return this;\n" + // + "}"; + } + + private static String privateLeafNodeSetter(LeafCNode n) { + if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n))) { + return ""; + } else { + return "\n\n" + // + "private Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "value) {\n" + // + " return " + n.getName() + "(" + builderType(n) + ".valueOf(" + INTERNAL_PREFIX + "value));\n" + // + "}"; + } + } + + private static String leafArraySetters(LeafCNode n) { + return publicLeafNodeSetters(n) + privateLeafNodeSetter(n); + } + + private static String innerMapSetters(CNode n) { + return "public Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "key, " + builderType(n) + " " + INTERNAL_PREFIX + + "value) {\n" + // + " " + n.getName() + ".put(" + INTERNAL_PREFIX + "key, " + INTERNAL_PREFIX + "value);\n" + // + " return this;\n" + // + "}\n" + // + "\n" + // + "public Builder " + n.getName() + "(Map<String, " + builderType(n) + "> " + INTERNAL_PREFIX + "values) {\n" + // + " " + n.getName() + ".putAll(" + INTERNAL_PREFIX + "values);\n" + // + " return this;\n" + // + "}"; + } + + private static String privateLeafMapSetter(CNode n) { + if ("String".equals(builderType(n)) || "FileReference".equals(builderType(n))) { + return ""; + } else { + return "\n\n" + // + "private Builder " + n.getName() + "(String " + INTERNAL_PREFIX + "key, String " + INTERNAL_PREFIX + "value) {\n" + // + " return " + n.getName() + "(" + INTERNAL_PREFIX + "key, " + builderType(n) + ".valueOf(" + INTERNAL_PREFIX + + "value));\n" + // + "}"; + } + } + + private static String leafMapSetters(CNode n) { + return innerMapSetters(n) + privateLeafMapSetter(n); + } + + private static String scalarSetters(LeafCNode n) { + String name = n.getName(); + + String signalInitialized = (n.getDefaultValue() == null) ? " " + INTERNAL_PREFIX + "uninitialized.remove(\"" + name + "\");\n" + : ""; + + String bType = builderType(n); + String stringSetter = "String".equals(bType) || "FileReference".equals(bType) ? "" + : String.format("\nprivate Builder %s(String %svalue) {\n" + // + " return %s(%s.valueOf(%svalue));\n" + // + "}", name, INTERNAL_PREFIX, name, boxedDataType(n), INTERNAL_PREFIX); + + String getNullGuard = bType.equals(boxedBuilderType(n)) ? String.format( + "\nif (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");", INTERNAL_PREFIX) : ""; + + return String.format("public Builder %s(%s %svalue) {%s\n" + // + " %s = %svalue;\n" + // + "%s", name, bType, INTERNAL_PREFIX, getNullGuard, name, INTERNAL_PREFIX, signalInitialized) + // + " return this;" + "\n}\n" + stringSetter; + } + } + + private static String setBuilderValueFromConfig(CNode child, CNode node) { + final String name = child.getName(); + final boolean isArray = child.isArray; + final boolean isMap = child.isMap; + + if (child instanceof FileLeaf && isArray) { + return name + "(" + userDataType(child) + ".toValues(config." + name + "()));"; + } else if (child instanceof FileLeaf && isMap) { + return name + "(" + userDataType(child) + ".toValueMap(config." + name + "()));"; + } else if (child instanceof FileLeaf) { + return name + "(config." + name + "().value());"; + } else if (child instanceof PathLeaf && isArray) { + return name + "(" + nodeClass(child) + ".toFileReferences(config." + name + "));"; + } else if (child instanceof PathLeaf && isMap) { + return name + "(" + nodeClass(child) + ".toFileReferenceMap(config." + name + "));"; + } else if (child instanceof PathLeaf) { + return name + "(config." + name + ".getFileReference());"; + } else if (child instanceof LeafCNode) { + return name + "(config." + name + "());"; + } else if (child instanceof InnerCNode && isArray) { + return setInnerArrayBuildersFromConfig((InnerCNode) child, node); + } else if (child instanceof InnerCNode && isMap) { + return setInnerMapBuildersFromConfig((InnerCNode) child); + } else { + return name + "(new " + builderType(child) + "(config." + name + "()));"; + } + } + + private static String setInnerArrayBuildersFromConfig(InnerCNode innerArr, CNode node) { + final String elemName = createUniqueSymbol(node, innerArr.getName()); + + return "for (" + userDataType(innerArr) + " " + elemName + " : config." + innerArr.getName() + "()) {\n" + // + " " + innerArr.getName() + "(new " + builderType(innerArr) + "(" + elemName + "));\n" + // + "}"; + } + + private static String setInnerMapBuildersFromConfig(InnerCNode innerMap) { + final String entryName = INTERNAL_PREFIX + "entry"; + return "for (Map.Entry<String, " + userDataType(innerMap) + "> " + entryName + " : config." + innerMap.getName() + + "().entrySet()) {\n" + // + " " + innerMap.getName() + "(" + entryName + ".getKey(), new " + userDataType(innerMap) + ".Builder(" + entryName + + ".getValue()));\n" + // + "}"; + } + + private static String getBuilderConstructors(CNode node, String className) { + return "public Builder() { }\n" + // + "\n" + // + "public Builder(" + className + " config) {\n" + // + indentCode(INDENTATION, + stream(node.getChildren()).map(child -> setBuilderValueFromConfig(child, node)).collect(Collectors.joining("\n"))) + + // + "\n}"; + } + + private static String conditionStatement(CNode child) { + final String superior = INTERNAL_PREFIX + "superior"; + + if (child.isArray) { + return "if (!" + superior + "." + child.getName() + ".isEmpty())"; + } else if (child.isMap) { + return ""; + } else if (child instanceof LeafCNode) { + return "if (" + superior + "." + child.getName() + " != null)"; + } else { + return ""; + } + } + + private static String overrideBuilderValue(CNode child) { + final String superior = INTERNAL_PREFIX + "superior"; + final String method = "override"; + final String name = child.getName(); + final String callSetter = name + "(" + superior + "." + name + ");"; + + if (child.isArray) { + String arrayOverride = INDENTATION + name + ".addAll(" + superior + "." + name + ");"; + return conditionStatement(child) + "\n" + arrayOverride; + } else if (child instanceof InnerCNode && !child.isArray && !child.isMap) { + return name + "(" + name + "." + method + "(" + superior + "." + name + "));"; + } else if (child.isMap) { + return callSetter; + } else { + return conditionStatement(child) + "\n" + INDENTATION + callSetter; + } + } + + private static String getOverrideMethod(CNode node) { + final String superior = INTERNAL_PREFIX + "superior"; + final String method = "override"; + + return "private Builder " + method + "(Builder " + superior + ") {\n" + // + indentCode(INDENTATION, + stream(node.getChildren()).map(BuilderGenerator::overrideBuilderValue).collect(Collectors.joining("\n"))) + + "\n" + // + " return this;\n" + // + "}"; + } + + private static String builderType(CNode node) { + if (node instanceof InnerCNode) { + return boxedDataType(node) + ".Builder"; + } else if (node instanceof FileLeaf) { + return "String"; + } else if (node instanceof PathLeaf) { + return "FileReference"; + } else if (node instanceof LeafCNode && (node.isArray || node.isMap)) { + return boxedDataType(node); + } else { + return userDataType(node); + } + } + + private static String boxedBuilderType(LeafCNode node) { + if (node instanceof FileLeaf) { + return "String"; + } else if (node instanceof PathLeaf) { + return "FileReference"; + } else { + return boxedDataType(node); + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java new file mode 100644 index 00000000000..9980cf565b1 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/ConfigGenerator.java @@ -0,0 +1,444 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import com.yahoo.config.codegen.LeafCNode.BooleanLeaf; +import com.yahoo.config.codegen.LeafCNode.DoubleLeaf; +import com.yahoo.config.codegen.LeafCNode.EnumLeaf; +import com.yahoo.config.codegen.LeafCNode.FileLeaf; +import com.yahoo.config.codegen.LeafCNode.IntegerLeaf; +import com.yahoo.config.codegen.LeafCNode.LongLeaf; +import com.yahoo.config.codegen.LeafCNode.PathLeaf; +import com.yahoo.config.codegen.LeafCNode.ReferenceLeaf; +import com.yahoo.config.codegen.LeafCNode.StringLeaf; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +import static com.yahoo.config.codegen.BuilderGenerator.getBuilder; +import static com.yahoo.config.codegen.JavaClassBuilder.INDENTATION; +import static com.yahoo.config.codegen.ReservedWords.INTERNAL_PREFIX; +import static java.util.Arrays.stream; + +/** + * @author gjoranv + * @author Tony Vaagenes + * @author ollivir + */ +public class ConfigGenerator { + // TODO: don't take indent as method param - the caller should indent + public static String generateContent(String indent, InnerCNode node, boolean isOuter) { + CNode[] children = node.getChildren(); + + return indentCode(indent, + getBuilder(node) + "\n\n" + + stream(children).map(ConfigGenerator::getFieldDefinition).collect(Collectors.joining("\n")) + "\n\n" + + getConstructors(node) + "\n\n" + + getAccessors(children) + "\n\n" + + getGetChangesRequiringRestart(node) + "\n\n" + + getContainsFieldsFlaggedWithRestart(node, isOuter) + + getStaticMethods(node) + + generateCodeForChildren(children, indent) + ); + } + + private static String generateCodeForChildren(CNode[] children, String indent) { + List<String> pieces = new LinkedList<>(); + for (CNode child : children) { + if (child instanceof EnumLeaf) { + pieces.add(getEnumCode((EnumLeaf) child) + "\n"); + } else if (child instanceof InnerCNode) { + pieces.add(getInnerDefinition((InnerCNode) child, indent) + "\n"); + } + } + return String.join("\n", pieces); + } + + private static String getInnerDefinition(InnerCNode inner, String indent) { + return (getClassDoc(inner) + "\n" +// + getClassDeclaration(inner) + "\n" +// + generateContent(indent, inner, false)).trim() + "\n}"; + } + + private static String getClassDeclaration(CNode node) { + return "public final static class " + nodeClass(node) + " extends InnerNode { \n"; + } + + private static String getFieldDefinition(CNode node) { + String fieldDef; + if (node instanceof LeafCNode && node.isArray) { + fieldDef = String.format("LeafNodeVector<%s, %s> %s;", boxedDataType(node), nodeClass(node), node.getName()); + } else if (node instanceof InnerCNode && node.isArray) { + fieldDef = String.format("InnerNodeVector<%s> %s;", nodeClass(node), node.getName()); + } else if (node.isMap) { + fieldDef = String.format("Map<String, %s> %s;", nodeClass(node), node.getName()); + } else { + fieldDef = String.format("%s %s;", nodeClass(node), node.getName()); + } + return node.getCommentBlock("//") + "private final " + fieldDef; + } + + private static String getStaticMethods(InnerCNode node) { + if (node.isArray) { + return getStaticMethodsForInnerArray(node) + "\n\n"; + } else if (node.isMap) { + return getStaticMethodsForInnerMap(node) + "\n\n"; + } else { + return ""; + } + } + + private static String getContainsFieldsFlaggedWithRestart(CNode node, boolean isOuter) { + if (isOuter) { + return String.format("private static boolean containsFieldsFlaggedWithRestart() {\n" +// + " return %b;\n" +// + "}\n\n", node.needRestart()); + } else { + return ""; + } + } + + private static String getGetChangesRequiringRestart(InnerCNode node) { + List<String> comparisons = new LinkedList<>(); + for (CNode child : node.getChildren()) { + if (child.needRestart()) { + comparisons.add("\n " + getComparison(child)); + } + } + + return "private ChangesRequiringRestart getChangesRequiringRestart(" + nodeClass(node) + " newConfig) {\n" +// + " ChangesRequiringRestart changes = new ChangesRequiringRestart(\"" + node.getName() + "\");" + String.join("", comparisons) + "\n" +// + " return changes;\n" +// + "}"; + } + + private static String quotedComment(CNode node) { + return node.getComment().replace("\n", "\\n").replace("\"", "\\\""); + } + + private static String getComparison(CNode node) { + if (node instanceof InnerCNode && node.isArray) { + return " changes.compareArray(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +// + " (a,b) -> ((" + nodeClass(node) + ")a).getChangesRequiringRestart((" + nodeClass(node) + ")b));"; + } else if (node instanceof InnerCNode && node.isMap) { + return " changes.compareMap(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +// + " (a,b) -> ((" + nodeClass(node) + ")a).getChangesRequiringRestart((" + nodeClass(node) + ")b));"; + } else if (node instanceof InnerCNode) { + return " changes.mergeChanges(\"" + node.getName() + "\", this." + node.getName() + ".getChangesRequiringRestart(newConfig." + node.getName() + "));"; + } else if (node.isArray) { + return " changes.compareArray(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +// + " (a,b) -> new ChangesRequiringRestart(\"" + node.getName() + "\").compare(a,b,\"\",\"" + quotedComment(node) + "\"));"; + } else if (node.isMap) { + return " changes.compareMap(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\",\n" +// + " (a,b) -> new ChangesRequiringRestart(\"" + node.getName() + "\").compare(a,b,\"\",\"" + quotedComment(node) + "\"));"; + } else { + return " changes.compare(this." + node.getName() + ", newConfig." + node.getName() + ", \"" + node.getName() + "\", \"" + quotedComment(node) + "\");"; + } + } + + private static String scalarDefault(LeafCNode scalar) { + if (scalar.getDefaultValue() == null) { + return ""; + } else if (scalar instanceof EnumLeaf && scalar.getDefaultValue().getValue() == null) { + return ""; + } else if (scalar instanceof EnumLeaf) { + return nodeClass(scalar) + "." + scalar.getDefaultValue().getStringRepresentation(); + } else if (scalar instanceof LongLeaf) { + return scalar.getDefaultValue().getStringRepresentation() + "L"; + } else if (scalar instanceof DoubleLeaf) { + return scalar.getDefaultValue().getStringRepresentation() + "D"; + } else { + return scalar.getDefaultValue().getStringRepresentation(); + } + } + + private static String assignFromBuilder(CNode child) { + final String name = child.getName(); + final String className = nodeClass(child); + final boolean isArray = child.isArray; + final boolean isMap = child.isMap; + + if (child instanceof FileLeaf && isArray) { + return name + " = LeafNodeVector.createFileNodeVector(builder." + name + ");"; + } else if (child instanceof PathLeaf && isArray) { + return name + " = LeafNodeVector.createPathNodeVector(builder." + name + ");"; + } else if (child instanceof LeafCNode && isArray) { + return name + " = new LeafNodeVector<>(builder." + name + ", new " + className + "());"; + } else if (child instanceof FileLeaf && isMap) { + return name + " = LeafNodeMaps.asFileNodeMap(builder." + name + ");"; + } else if (child instanceof PathLeaf && isMap) { + return name + " = LeafNodeMaps.asPathNodeMap(builder." + name + ");"; + } else if (child instanceof LeafCNode && isMap) { + return name + " = LeafNodeMaps.asNodeMap(builder." + name + ", new " + className + "());"; + } else if (child instanceof InnerCNode && isArray) { + return name + " = " + className + ".createVector(builder." + name + ");"; + } else if (child instanceof InnerCNode && isMap) { + return name + " = " + className + ".createMap(builder." + name + ");"; + } else if (child instanceof InnerCNode) { + return name + " = new " + className + "(builder." + name + ", throwIfUninitialized);"; + } else if (child instanceof LeafCNode) { + return name + " = (builder." + name + " == null) ?\n" +// + " new " + className + "(" + scalarDefault((LeafCNode) child) + ") : new " + className + "(builder." + name + ");"; + } else { + throw new IllegalStateException("Cannot create assignment for node"); // should not happen + } + } + + private static String getConstructors(InnerCNode inner) { + // TODO: merge these two constructors into one when the config library uses builders to set values from payload. + return "public " + nodeClass(inner) + "(Builder builder) {\n" +// + " this(builder, true);\n" +// + "}\n" +// + "\n" +// + "private " + nodeClass(inner) + "(Builder builder, boolean throwIfUninitialized) {\n" +// + " if (throwIfUninitialized && ! builder." + INTERNAL_PREFIX + "uninitialized.isEmpty())\n" +// + " throw new IllegalArgumentException(\"The following builder parameters for \" +\n" +// + " \"" + inner.getFullName() + " must be initialized: \" + builder." + INTERNAL_PREFIX + "uninitialized);\n" +// + "\n" +// + indentCode(INDENTATION, stream(inner.getChildren()).map(ConfigGenerator::assignFromBuilder).collect(Collectors.joining("\n"))) + "\n" +// + "}"; + } + + private static String getAccessorCode(CNode node) { + if (node.isArray) { + return accessorsForArray(node); + } else if (node.isMap) { + return accessorsForMap(node); + } else { + return accessorForStructOrScalar(node); + } + } + + private static String valueAccessor(CNode node) { + if (node instanceof LeafCNode) { + return ".value()"; + } else { + return ""; + } + } + + private static String listAccessor(CNode node) { + if (node instanceof LeafCNode) { + return node.getName() + ".asList()"; + } else { + return node.getName(); + } + } + + private static String mapAccessor(CNode node) { + if (node instanceof LeafCNode) { + return "LeafNodeMaps.asValueMap(" + node.getName() + ")"; + } else { + return "Collections.unmodifiableMap(" + node.getName() + ")"; + } + } + + private static String accessorsForArray(CNode node) { + final String name = node.getName(); + final String fullName = node.getFullName(); + return "/**\n" +// + " * @return " + fullName + "\n" +// + " */\n" +// + "public List<" + boxedDataType(node) + "> " + name + "() {\n" +// + " return " + listAccessor(node) + ";\n" +// + "}\n" +// + "\n" +// + "/**\n" +// + " * @param i the index of the value to return\n" +// + " * @return " + fullName + "\n" +// + " */\n" +// + "public " + userDataType(node) + " " + name + "(int i) {\n" +// + " return " + name + ".get(i)" + valueAccessor(node) + ";\n" +// + "}"; + } + + private static String accessorsForMap(CNode node) { + final String name = node.getName(); + final String fullName = node.getFullName(); + + return "/**\n" +// + " * @return " + fullName + "\n" +// + " */\n" +// + "public Map<String, " + boxedDataType(node) + "> " + name + "() {\n" +// + " return " + mapAccessor(node) + ";\n" +// + "}\n" +// + "\n" +// + "/**\n" +// + " * @param key the key of the value to return\n" +// + " * @return " + fullName + "\n" +// + " */\n" +// + "public " + userDataType(node) + " " + name + "(String key) {\n" +// + " return " + name + ".get(key)" + valueAccessor(node) + ";\n" +// + "}"; + } + + private static String accessorForStructOrScalar(CNode node) { + return "/**\n" +// + " * @return " + node.getFullName() + "\n" +// + " */\n" +// + "public " + userDataType(node) + " " + node.getName() + "() {\n" +// + " return " + node.getName() + valueAccessor(node) + ";\n" +// + "}"; + } + + private static String getAccessors(CNode[] children) { + List<String> accessors = new LinkedList<>(); + for (CNode child : children) { + String accessor = getAccessorCode(child); + if (accessor.isEmpty() == false) { + accessors.add(accessor); + } + } + return String.join("\n\n", accessors); + } + + private static String getStaticMethodsForInnerArray(InnerCNode inner) { + final String nc = nodeClass(inner); + return String.format("private static InnerNodeVector<%s> createVector(List<Builder> builders) {\n" +// + " List<%s> elems = new ArrayList<>();\n" +// + " for (Builder b : builders) {\n" +// + " elems.add(new %s(b));\n" +// + " }\n" +// + " return new InnerNodeVector<%s>(elems);\n" +// + "}", nc, nc, nc, nc); + } + + private static String getStaticMethodsForInnerMap(InnerCNode inner) { + final String nc = nodeClass(inner); + return String.format( + "private static Map<String, %s> createMap(Map<String, Builder> builders) {\n" +// + " Map<String, %s> ret = new LinkedHashMap<>();\n" +// + " for(String key : builders.keySet()) {\n" +// + " ret.put(key, new %s(builders.get(key)));\n" +// + " }\n" +// + " return Collections.unmodifiableMap(ret);\n" +// + "}", nc, nc, nc); + } + + private static String getEnumCode(EnumLeaf en) { + String enumValues = stream(en.getLegalValues()).map(e -> String.format(" public final static Enum %s = Enum.%s;", e, e)).collect(Collectors.joining("\n")); + + String code = String.format("%s\n" +// + "public final static class %s extends EnumNode<%s> {\n" +// + "\n" +// + " public %s(){\n" +// + " this.value = null;\n" +// + " }\n" +// + "\n" +// + " public %s(Enum enumValue) {\n" +// + " super(enumValue != null);\n" +// + " this.value = enumValue;\n" +// + " }\n" +// + "\n" +// + " public enum Enum {%s}\n" +// + "%s\n" +// + "\n" +// + " @Override\n" +// + " protected boolean doSetValue(@NonNull String name) {\n" +// + " try {\n" +// + " value = Enum.valueOf(name);\n" +// + " return true;\n" +// + " } catch (IllegalArgumentException e) {\n" +// + " }\n" +// + " return false;\n" +// + " }\n" +// + "}", getClassDoc(en), + nodeClass(en), + nodeClass(en) + ".Enum", + nodeClass(en), + nodeClass(en), + String.join(", ", en.getLegalValues()), + enumValues); + + return indentCode("", code); + } + + private static String getClassDoc(CNode node) { + String header = "/**\n" + " * This class represents " + node.getFullName(); + String nodeComment = node.getCommentBlock(" *"); + if (nodeComment.isEmpty()) { + return header + "\n */"; + } else { + if (nodeComment.endsWith("\n")) { + nodeComment = nodeComment.substring(0, nodeComment.length() - 1); + } + return header + "\n * \n" + nodeComment + "\n */"; + } + } + + static String indentCode(String indent, String code) { + List<String> indented = new LinkedList<>(); + for (String line : code.split("\n", -1)) { + indented.add(line.length() > 0 ? indent + line : line); + } + return String.join("\n", indented); + } + + /** + * @return the name of the class that is generated by this node. + */ + static String nodeClass(CNode node) { + if (node.getName().length() == 0) { + throw new CodegenRuntimeException("Node with empty name, under parent " + node.getParent().getName()); + } else if (node instanceof InnerCNode && node.getParent() == null) { + return ConfiggenUtil.createClassName(node.getName()); + } else if (node instanceof BooleanLeaf) { + return "BooleanNode"; + } else if (node instanceof DoubleLeaf) { + return "DoubleNode"; + } else if (node instanceof FileLeaf) { + return "FileNode"; + } else if (node instanceof PathLeaf) { + return "PathNode"; + } else if (node instanceof IntegerLeaf) { + return "IntegerNode"; + } else if (node instanceof LongLeaf) { + return "LongNode"; + } else if (node instanceof ReferenceLeaf) { + return "ReferenceNode"; + } else if (node instanceof StringLeaf) { + return "StringNode"; + } else { + return ConfiggenUtil.capitalize(node.getName()); + } + } + + static String userDataType(CNode node) { + if (node instanceof InnerCNode) { + return nodeClass(node); + } else if (node instanceof EnumLeaf) { + return nodeClass(node) + ".Enum"; + } else if (node instanceof BooleanLeaf) { + return "boolean"; + } else if (node instanceof DoubleLeaf) { + return "double"; + } else if (node instanceof FileLeaf) { + return "FileReference"; + } else if (node instanceof PathLeaf) { + return "Path"; + } else if (node instanceof IntegerLeaf) { + return "int"; + } else if (node instanceof LongLeaf) { + return "long"; + } else if (node instanceof StringLeaf) { + return "String"; + } else { + throw new IllegalStateException("Cannot determine user data type for node"); // should not occur + } + } + + /** + * @return the boxed java data type, e.g. Integer for int + */ + static String boxedDataType(CNode node) { + String rawType = userDataType(node); + + if ("int".equals(rawType)) { + return "Integer"; + } else if (rawType.toLowerCase().equals(rawType)) { + return ConfiggenUtil.capitalize(rawType); + } else { + return rawType; + } + } +} diff --git a/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java b/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java index 299a5540098..995ef419f30 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/ConfiggenUtil.java @@ -26,7 +26,7 @@ public class ConfiggenUtil { return className; } - private static String capitalize(String in) { + static String capitalize(String in) { StringBuilder sb = new StringBuilder(in); sb.setCharAt(0, Character.toTitleCase(in.charAt(0))); return sb.toString(); diff --git a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java new file mode 100644 index 00000000000..00498094db5 --- /dev/null +++ b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java @@ -0,0 +1,170 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.yahoo.config.codegen.ConfigGenerator.indentCode; +import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; +import static com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX; + +/** + * Builds one Java class based on the given CNode tree. + * + * @author gjoranv + * @author Tony Vaagenes + * @author ollivir + */ +public class JavaClassBuilder implements ClassBuilder { + public static final String INDENTATION = " "; + + private final InnerCNode root; + private final NormalizedDefinition nd; + private final String packagePrefix; + private final String javaPackage; + private final String className; + private final File destDir; + + public JavaClassBuilder(InnerCNode root, NormalizedDefinition nd, File destDir, String rawPackagePrefix) { + this.root = root; + this.nd = nd; + this.packagePrefix = (rawPackagePrefix != null) ? rawPackagePrefix : DEFAULT_PACKAGE_PREFIX; + this.javaPackage = (root.getPackage() != null) ? root.getPackage() : packagePrefix + root.getNamespace(); + this.className = createClassName(root.getName()); + this.destDir = destDir; + } + + @Override + public void createConfigClasses() { + try { + File outFile = new File(getDestPath(destDir, javaPackage), className + ".java"); + try (PrintStream out = new PrintStream(new FileOutputStream(outFile))) { + out.print(getConfigClass(className)); + } + System.err.println(outFile.getPath() + " successfully written."); + } catch (FileNotFoundException e) { + throw new CodegenRuntimeException(e); + } + } + + public String getConfigClass(String className) { + return getHeader() + "\n\n" + // + getRootClassDeclaration(root, className) + "\n\n" + // + indentCode(INDENTATION, getFrameworkCode()) + "\n\n" + // + ConfigGenerator.generateContent(INDENTATION, root, true) + "\n" + // + "}\n"; + } + + private String getHeader() { + return "/**\n" + // + " * This file is generated from a config definition file.\n" + // + " * ------------ D O N O T E D I T ! ------------\n" + // + " */\n" + // + "\n" + // + "package " + javaPackage + ";\n" + // + "\n" + // + "import java.util.*;\n" + // + "import java.nio.file.Path;\n" + // + "import edu.umd.cs.findbugs.annotations.NonNull;\n" + // + getImportFrameworkClasses(root.getNamespace()); + } + + private String getImportFrameworkClasses(String namespace) { + if (CNode.DEFAULT_NAMESPACE.equals(namespace) == false) { + return "import " + packagePrefix + CNode.DEFAULT_NAMESPACE + ".*;"; + } else { + return ""; + } + } + + // TODO: remove the extra comment line " *" if root.getCommentBlock is empty + private String getRootClassDeclaration(InnerCNode root, String className) { + return "/**\n" + // + " * This class represents the root node of " + root.getFullName() + "\n" + // + " *\n" + // + "" + root.getCommentBlock(" *") + " */\n" + // + "public final class " + className + " extends ConfigInstance {\n" + // + "\n" + // + " public final static String CONFIG_DEF_MD5 = \"" + root.getMd5() + "\";\n" + // + " public final static String CONFIG_DEF_NAME = \"" + root.getName() + "\";\n" + // + " public final static String CONFIG_DEF_NAMESPACE = \"" + root.getNamespace() + "\";\n" + // + " public final static String CONFIG_DEF_VERSION = \"" + root.getVersion() + "\";\n" + // + " public final static String[] CONFIG_DEF_SCHEMA = {\n" + // + "" + indentCode(INDENTATION + INDENTATION, getDefSchema()) + "\n" + // + " };\n" + // + "\n" + // + " public static String getDefMd5() { return CONFIG_DEF_MD5; }\n" + // + " public static String getDefName() { return CONFIG_DEF_NAME; }\n" + // + " public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }\n" + // + " public static String getDefVersion() { return CONFIG_DEF_VERSION; }"; + } + + private String getDefSchema() { + return nd.getNormalizedContent().stream().map(l -> "\"" + l.replace("\"", "\\\"") + "\"").collect(Collectors.joining(",\n")); + } + + private String getFrameworkCode() { + return "public interface Producer extends ConfigInstance.Producer {\n" + // + " void getConfig(Builder builder);\n" + // + "}"; + } + + /** + * @param rootDir + * The root directory for the destination path. + * @param javaPackage + * The java package + * @return the destination path for the generated config file, including the + * given rootDir. + */ + private File getDestPath(File rootDir, String javaPackage) { + File dir = rootDir; + for (String subDir : javaPackage.split("\\.")) { + dir = new File(dir, subDir); + synchronized (this) { + if (!dir.isDirectory() && !dir.mkdir()) { + throw new CodegenRuntimeException("Could not create " + dir.getPath()); + } + } + } + return dir; + } + + /** + * Returns a name that can be safely used as a local variable in the generated + * config class for the given node. The name will be based on the given basis + * string, but the basis itself is not a possible return value. + * + * @param node + * The node to find a unused symbol name for. + * @param basis + * The basis for the generated symbol name. + * @return A name that is not used in the given config node. + */ + static String createUniqueSymbol(CNode node, String basis) { + Set<String> usedSymbols = Arrays.stream(node.getChildren()).map(CNode::getName).collect(Collectors.toSet()); + Random rng = new Random(); + + for (int i = 1;; i++) { + String candidate = (i < basis.length()) ? basis.substring(0, i) + : ReservedWords.INTERNAL_PREFIX + basis + rng.nextInt(Integer.MAX_VALUE); + if (usedSymbols.contains(candidate) == false) { + return candidate; + } + } + } + + public String className() { + return className; + } + + public String javaPackage() { + return javaPackage; + } +} diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala deleted file mode 100644 index 4f6f310e32e..00000000000 --- a/configgen/src/main/scala/com/yahoo/config/codegen/BuilderGenerator.scala +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.codegen - -import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix} -import JavaClassBuilder.{Indentation, createUniqueSymbol} -import ConfigGenerator.{indentCode, nodeClass, userDataType, boxedDataType} -import com.yahoo.config.codegen.LeafCNode._ - -/** - * @author gjoranv - */ - -object BuilderGenerator { - - def getBuilder(node: InnerCNode): String = { - getDeclaration(node) + "\n" + - indentCode(Indentation, - getUninitializedScalars(node) + "\n\n" + - node.getChildren.map(getBuilderFieldDefinition).mkString("\n") + "\n\n" + - getBuilderConstructors(node, nodeClass(node)) + "\n\n" + - getOverrideMethod(node) + "\n\n" + - getBuilderSetters(node) + "\n" + - getSpecialRootBuilderCode(node) - ) + - "}" - } - - private def getDeclaration(node: InnerCNode) = { - def getInterfaces = - if (node.getParent == null) "implements ConfigInstance.Builder" - else "implements ConfigBuilder" - - "public static class Builder " + getInterfaces + " {" - } - - private def getSpecialRootBuilderCode(node: InnerCNode) = { - if (node.getParent == null) "\n" + getDispatchCode(node) + "\n" - else "" - } - - private def getDispatchCode(node: InnerCNode) = { - // Use full path to @Override, as users are free to define an inner node called 'override'. (summarymap.def does) - // The generated inner 'Override' class would otherwise be mistaken for the annotation. - """ - |@java.lang.Override - |public final boolean dispatchGetConfig(ConfigInstance.Producer producer) { - | if (producer instanceof Producer) { - | ((Producer)producer).getConfig(this); - | return true; - | } - | return false; - |} - | - |@java.lang.Override - |public final String getDefMd5() { return CONFIG_DEF_MD5; } - |@java.lang.Override - |public final String getDefName() { return CONFIG_DEF_NAME; } - |@java.lang.Override - |public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } - """.stripMargin.trim - } - - private def getUninitializedScalars(node: InnerCNode): String = { - val scalarsWithoutDefault = { - node.getChildren.collect { - case leaf: LeafCNode if (!leaf.isArray && !leaf.isMap && leaf.getDefaultValue == null) => - "\"" + leaf.getName + "\"" - } - } - - val uninitializedList = - if (scalarsWithoutDefault.size > 0) - "Arrays.asList(\n" + indentCode(Indentation, scalarsWithoutDefault.mkString("",",\n","\n)")) - else - "" - - "private Set<String> " + InternalPrefix + "uninitialized = new HashSet<String>(" + uninitializedList + ");" - } - - private def getBuilderFieldDefinition(node: CNode): String = { - - (node match { - case array if node.isArray => - "public List<%s> %s = new ArrayList<>()".format(builderType(array), array.getName) - case map if node.isMap => - "public Map<String, %s> %s = new LinkedHashMap<>()".format(builderType(map), map.getName) - case struct: InnerCNode => - "public %s %s = new %s()".format(builderType(struct), struct.getName, builderType(struct)) - case scalar : LeafCNode => - "private " + boxedBuilderType(scalar) + " " + scalar.getName + " = null" - }) + ";" - } - - private def getBuilderSetters(node: CNode): String = { - val children: Array[CNode] = node.getChildren - - def structSetter(node: InnerCNode) = { - <code> - |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{ - | {node.getName} = {InternalPrefix}builder; - | return this; - |}} - </code>.text.stripMargin.trim - } - - def innerArraySetters(node: InnerCNode) = { - <code> - |/** - | * Add the given builder to this builder's list of {nodeClass(node)} builders - | * @param {InternalPrefix}builder a builder - | * @return this builder - | */ - |public Builder {node.getName}({builderType(node)} {InternalPrefix}builder) {{ - | {node.getName}.add({InternalPrefix}builder); - | return this; - |}} - | - |/** - | * Set the given list as this builder's list of {nodeClass(node)} builders - | * @param __builders a list of builders - | * @return this builder - | */ - |public Builder {node.getName}(List<{builderType(node)}> __builders) {{ - | {node.getName} = __builders; - | return this; - |}} - </code>.text.stripMargin.trim - } - - def leafArraySetters(node: LeafCNode) = { - val setters = - <code> - |public Builder {node.getName}({builderType(node)} {InternalPrefix}value) {{ - | {node.getName}.add({InternalPrefix}value); - | return this; - |}} - | - |public Builder {node.getName}(Collection<{builderType(node)}> {InternalPrefix}values) {{ - | {node.getName}.addAll({InternalPrefix}values); - | return this; - |}} - </code>.text.stripMargin.trim - - val privateSetter = - if (builderType(node) == "String" || builderType(node) == "FileReference") - "" - else - "\n\n" + - <code> - | - | - |private Builder {node.getName}(String {InternalPrefix}value) {{ - | return {node.getName}({builderType(node)}.valueOf({InternalPrefix}value)); - |}} - </code>.text.stripMargin.trim - - setters + privateSetter - } - - def innerMapSetters(node: CNode) = { - <code> - |public Builder {node.getName}(String {InternalPrefix}key, {builderType(node)} {InternalPrefix}value) {{ - | {node.getName}.put({InternalPrefix}key, {InternalPrefix}value); - | return this; - |}} - | - |public Builder {node.getName}(Map<String, {builderType(node)}> {InternalPrefix}values) {{ - | {node.getName}.putAll({InternalPrefix}values); - | return this; - |}} - </code>.text.stripMargin.trim - } - - def leafMapSetters(node: LeafCNode) = { - val privateSetter = - if (builderType(node) == "String" || builderType(node) == "FileReference") - "" - else - "\n\n" + - <code> - | - | - |private Builder {node.getName}(String {InternalPrefix}key, String {InternalPrefix}value) {{ - | return {node.getName}({InternalPrefix}key, {builderType(node)}.valueOf({InternalPrefix}value)); - |}} - </code>.text.stripMargin.trim - - innerMapSetters(node) + privateSetter - } - - def scalarSetters(node: LeafCNode): String = { - val name = node.getName - - val signalInitialized = - if (node.getDefaultValue == null) InternalPrefix + "uninitialized.remove(\"" + name + "\");\n" - else "" - - val stringSetter = - builderType(node) match { - case "String" => "" - case "FileReference" => "" - case _ => - """| - |private Builder %s(String %svalue) { - | return %s(%s.valueOf(%svalue)); - |}""".stripMargin.format(name, InternalPrefix, - name, boxedDataType(node), InternalPrefix) - } - - def getNullGuard = { - if (builderType(node) != boxedBuilderType(node)) - "" - else - "\n" + "if (%svalue == null) throw new IllegalArgumentException(\"Null value is not allowed.\");" - .format(InternalPrefix) - } - - // TODO: check if 2.9.2 allows string to start with a newline - """|public Builder %s(%s %svalue) {%s - | %s = %svalue; - | %s - """.stripMargin.format(name, builderType(node), InternalPrefix, getNullGuard, - name, InternalPrefix, - signalInitialized).trim + - "\n return this;" + "\n}\n" + - stringSetter - } - - (children collect { - case innerArray: InnerCNode if innerArray.isArray => innerArraySetters(innerArray) - case innerMap: InnerCNode if innerMap.isMap => innerMapSetters(innerMap) - case leafArray: LeafCNode if leafArray.isArray => leafArraySetters(leafArray) - case leafMap: LeafCNode if leafMap.isMap => leafMapSetters(leafMap) - case struct: InnerCNode => structSetter(struct) - case scalar: LeafCNode => scalarSetters(scalar) - } ).mkString("\n\n") - } - - private def getBuilderConstructors(node: CNode, className: String): String = { - def setBuilderValueFromConfig(child: CNode) = { - val name = child.getName - val isArray = child.isArray - val isMap = child.isMap - - child match { - case fileArray: FileLeaf if isArray => name + "(" + userDataType(fileArray) + ".toValues(config." + name + "()));" - case fileMap: FileLeaf if isMap => name + "(" + userDataType(fileMap) + ".toValueMap(config." + name + "()));" - case file: FileLeaf => name + "(config." + name + "().value());" - case pathArray: PathLeaf if isArray => name + "(" + nodeClass(pathArray) + ".toFileReferences(config." + name + "));" - case pathMap: PathLeaf if isMap => name + "(" + nodeClass(pathMap) + ".toFileReferenceMap(config." + name + "));" - case path: PathLeaf => name + "(config." + name + ".getFileReference());" - case leaf: LeafCNode => name + "(config." + name + "());" - case innerArray: InnerCNode if isArray => setInnerArrayBuildersFromConfig(innerArray) - case innerMap: InnerCNode if isMap => setInnerMapBuildersFromConfig(innerMap) - case struct => name + "(new " + builderType(struct) + "(config." + name + "()));" - } - } - - def setInnerArrayBuildersFromConfig(innerArr: InnerCNode) = { - val elemName = createUniqueSymbol(node, innerArr.getName) - <code> - |for ({userDataType(innerArr)} {elemName} : config.{innerArr.getName}()) {{ - | {innerArr.getName}(new {builderType(innerArr)}({elemName})); - |}} - </code>.text.stripMargin.trim - } - - def setInnerMapBuildersFromConfig(innerMap: InnerCNode) = { - val entryName = InternalPrefix + "entry" - <code> - |for (Map.Entry<String, {userDataType(innerMap)}> {entryName} : config.{innerMap.getName}().entrySet()) {{ - | {innerMap.getName}({entryName}.getKey(), new {userDataType(innerMap)}.Builder({entryName}.getValue())); - |}} - </code>.text.stripMargin.trim - } - - <code> - |public Builder() {{ }} - | - |public Builder({className} config) {{ - |{indentCode(Indentation, node.getChildren.map(setBuilderValueFromConfig).mkString("\n"))} - |}} - </code>.text.stripMargin.trim - } - - def arrayOverride(name: String, superior: String): String = { - Indentation + name + ".addAll(" + superior + "." + name + ");" - } - - private def getOverrideMethod(node:CNode): String = { - val method = "override" - val superior = InternalPrefix + "superior" - - def callSetter(name: String): String = { - name + "(" + superior + "." + name + ");" - } - def overrideBuilderValue(child: CNode) = { - val name = child.getName - child match { - case leafArray: CNode if (child.isArray) => - conditionStatement(child) + "\n" + arrayOverride(name, superior) - case struct: InnerCNode if !(child.isArray || child.isMap) => - name + "(" + name + "." + method + "(" + superior + "." + name + "));" - case map: CNode if child.isMap => - callSetter(name) - case _ => - conditionStatement(child) + "\n" + - Indentation + callSetter(name) - } - } - - def conditionStatement(child: CNode) = { - val name = child.getName - val isArray = child.isArray - val isMap = child.isMap - child match { - case _ if isArray => "if (!" + superior + "." + name + ".isEmpty())" - case _ if isMap => "" - case scalar: LeafCNode => "if (" + superior + "." + name + " != null)" - case struct => "" - } - } - - <code> - |private Builder {method}(Builder {superior}) {{ - |{indentCode(Indentation, node.getChildren.map(overrideBuilderValue).mkString("\n"))} - | return this; - |}} - </code>.text.stripMargin.trim - } - - private def builderType(node: CNode): String = { - node match { - case inner: InnerCNode => boxedDataType(node) + ".Builder" - case file: FileLeaf => "String" - case path: PathLeaf => "FileReference" - case leafArray: LeafCNode if (node.isArray || node.isMap) => boxedDataType(node) - case _ => userDataType(node) - } - } - - private def boxedBuilderType(node: LeafCNode): String = { - node match { - case file: FileLeaf => "String" - case path: PathLeaf => "FileReference" - case _ => boxedDataType(node) - } - } - -} diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala b/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala deleted file mode 100644 index 38306a03575..00000000000 --- a/configgen/src/main/scala/com/yahoo/config/codegen/ConfigGenerator.scala +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.codegen - - -import com.yahoo.config.codegen.BuilderGenerator.getBuilder -import com.yahoo.config.codegen.JavaClassBuilder.Indentation -import com.yahoo.config.codegen.LeafCNode._ -import com.yahoo.config.codegen.ReservedWords.{INTERNAL_PREFIX => InternalPrefix} - -/** - * @author gjoranv - * @author tonytv - */ -// TODO: don't take indent as method param - the caller should indent -object ConfigGenerator { - - def generateContent(indent: String, node: InnerCNode, isOuter: Boolean = true): String = { - val children: Array[CNode] = node.getChildren - - def generateCodeForChildren: String = { - (children collect { - case enum: EnumLeaf => getEnumCode(enum, "") + "\n" - case inner: InnerCNode => getInnerDefinition(inner, indent) + "\n" - } ).mkString("\n") - } - - def getInnerDefinition(inner: InnerCNode, indent: String) = { - <code> - |{getClassDoc(inner, indent)} - |{getClassDeclaration(inner)} - |{generateContent(indent, inner, false)} - </code>.text.stripMargin.trim + "\n}" - } - - def getClassDeclaration(node: CNode): String = { - "public final static class " + nodeClass(node)+ " extends InnerNode { " + "\n" - } - - def getFieldDefinition(node: CNode): String = { - node.getCommentBlock("//") + "private final " + - (node match { - case _: LeafCNode if node.isArray => - "LeafNodeVector<%s, %s> %s;".format(boxedDataType(node), nodeClass(node), node.getName) - case _: InnerCNode if node.isArray => - "InnerNodeVector<%s> %s;".format(nodeClass(node), node.getName) - case _ if node.isMap => - "Map<String, %s> %s;".format(nodeClass(node), node.getName) - case _ => - "%s %s;".format(nodeClass(node), node.getName) - }) - } - - def getStaticMethods = { - if (node.isArray) getStaticMethodsForInnerArray(node) + "\n\n" - else if (node.isMap) getStaticMethodsForInnerMap(node) + "\n\n" - else "" - } - - def getContainsFieldsFlaggedWithRestart(node: CNode): String = { - if (isOuter) { - """ - |private static boolean containsFieldsFlaggedWithRestart() { - | return %b; - |} - """.stripMargin.trim.format(node.needRestart) + "\n\n" - } else "" - } - - indentCode(indent, - getBuilder(node) + "\n\n" + - children.map(getFieldDefinition).mkString("\n") + "\n\n" + - getConstructors(node) + "\n\n" + - getAccessors(children) + "\n\n" + - getGetChangesRequiringRestart(node) + "\n\n" + - getContainsFieldsFlaggedWithRestart(node) + - getStaticMethods + - generateCodeForChildren - ) - } - - private def getGetChangesRequiringRestart(node: InnerCNode): String = { - def quotedComment(node: CNode): String = { - node.getComment.replace("\n", "\\n").replace("\"", "\\\"") - } - - def getComparison(node: CNode): String = node match { - case inner: InnerCNode if inner.isArray => - <code> - | changes.compareArray(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}", - | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b)); - </code>.text.stripMargin.trim - case inner: InnerCNode if inner.isMap => - <code> - | changes.compareMap(this.{inner.getName}, newConfig.{inner.getName}, "{inner.getName}", "{quotedComment(inner)}", - | (a,b) -> (({nodeClass(inner)})a).getChangesRequiringRestart(({nodeClass(inner)})b)); - </code>.text.stripMargin.trim - case inner: InnerCNode => - <code> - | changes.mergeChanges("{inner.getName}", this.{inner.getName}.getChangesRequiringRestart(newConfig.{inner.getName})); - </code>.text.stripMargin.trim - case node: CNode if node.isArray => - <code> - | changes.compareArray(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}", - | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}")); - </code>.text.stripMargin.trim - case node: CNode if node.isMap => - <code> - | changes.compareMap(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}", - | (a,b) -> new ChangesRequiringRestart("{node.getName}").compare(a,b,"","{quotedComment(node)}")); - </code>.text.stripMargin.trim - case node: CNode => - <code> - | changes.compare(this.{node.getName}, newConfig.{node.getName}, "{node.getName}", "{quotedComment(node)}"); - </code>.text.stripMargin.trim - } - - val comparisons = - for { - c <- node.getChildren if c.needRestart - } yield "\n " + getComparison(c) - - <code> - |private ChangesRequiringRestart getChangesRequiringRestart({nodeClass(node)} newConfig) {{ - | ChangesRequiringRestart changes = new ChangesRequiringRestart("{node.getName}");{comparisons.mkString("")} - | return changes; - |}} - </code>.text.stripMargin.trim - } - - - private def scalarDefault(scalar: LeafCNode): String = { - scalar match { - case _ if scalar.getDefaultValue == null => "" - case enumWithNullDefault: EnumLeaf if enumWithNullDefault.getDefaultValue.getValue == null => "" - case enum: EnumLeaf => nodeClass(enum) + "." + enum.getDefaultValue.getStringRepresentation - case long: LongLeaf => long.getDefaultValue.getStringRepresentation + "L" - case double: DoubleLeaf => double.getDefaultValue.getStringRepresentation + "D" - case _ => scalar.getDefaultValue.getStringRepresentation - } - } - - private def getConstructors(inner: InnerCNode) = { - - def assignFromBuilder(child: CNode) = { - val name = child.getName - val className = nodeClass(child) - val dataType = boxedDataType(child) - val isArray = child.isArray - val isMap = child.isMap - - def assignIfInitialized(leaf: LeafCNode) = { - <code> - |{name} = (builder.{name} == null) ? - | new {className}({scalarDefault(leaf)}) : new {className}(builder.{name}); - </code>.text.stripMargin.trim - } - - child match { - case fileArray: FileLeaf if isArray => - name + " = LeafNodeVector.createFileNodeVector(builder."+ name +");" - case pathArray: PathLeaf if isArray => - name + " = LeafNodeVector.createPathNodeVector(builder."+ name +");" - case leafArray: LeafCNode if isArray => - name + " = new LeafNodeVector<>(builder."+ name +", new " + className + "());" - case fileMap: LeafCNode if isMap && child.isInstanceOf[FileLeaf] => - name + " = LeafNodeMaps.asFileNodeMap(builder."+ name +");" - case pathMap: LeafCNode if isMap && child.isInstanceOf[PathLeaf] => - name + " = LeafNodeMaps.asPathNodeMap(builder."+ name +");" - case leafMap: LeafCNode if isMap => - name + " = LeafNodeMaps.asNodeMap(builder."+ name +", new " + className + "());" - case innerArray: InnerCNode if isArray => - name + " = " + className + ".createVector(builder." + name + ");" - case innerMap: InnerCNode if isMap => - name + " = " + className + ".createMap(builder." + name + ");" - case struct: InnerCNode => - name + " = new " + className + "(builder." + name + ", throwIfUninitialized);" - case leaf: LeafCNode => - assignIfInitialized(leaf) - } - } - - // TODO: merge these two constructors into one when the config library uses builders to set values from payload. - <code> - |public {nodeClass(inner)}(Builder builder) {{ - | this(builder, true); - |}} - | - |private {nodeClass(inner)}(Builder builder, boolean throwIfUninitialized) {{ - | if (throwIfUninitialized && ! builder.{InternalPrefix}uninitialized.isEmpty()) - | throw new IllegalArgumentException("The following builder parameters for " + - | "{inner.getFullName} must be initialized: " + builder.{InternalPrefix}uninitialized); - | - |{indentCode(Indentation, inner.getChildren.map(assignFromBuilder).mkString("\n"))} - |}} - </code>.text.stripMargin.trim - } - - private def getAccessors(children: Array[CNode]): String = { - - def getAccessorCode(indent: String, node: CNode): String = { - indentCode(indent, - if (node.isArray) - accessorsForArray(node) - else if (node.isMap) - accessorsForMap(node) - else - accessorForStructOrScalar(node)) - } - - def valueAccessor(node: CNode) = node match { - case leaf: LeafCNode => ".value()" - case inner => "" - } - - def listAccessor(node: CNode) = node match { - case leaf: LeafCNode => "%s.asList()".format(leaf.getName) - case inner => inner.getName - } - - def mapAccessor(node: CNode) = node match { - case leaf: LeafCNode => "LeafNodeMaps.asValueMap(%s)".format(leaf.getName) - case inner => "Collections.unmodifiableMap(%s)".format(inner.getName) - } - - def accessorsForArray(node: CNode): String = { - val name = node.getName - val fullName = node.getFullName - <code> - |/** - | * @return {fullName} - | */ - |public List<{boxedDataType(node)}> {name}() {{ - | return {listAccessor(node)}; - |}} - | - |/** - | * @param i the index of the value to return - | * @return {fullName} - | */ - |public {userDataType(node)} {name}(int i) {{ - | return {name}.get(i){valueAccessor(node)}; - |}} - </code>.text.stripMargin.trim - } - - def accessorsForMap(node: CNode): String = { - val name = node.getName - val fullName = node.getFullName - <code> - |/** - | * @return {fullName} - | */ - |public Map<String, {boxedDataType(node)}> {name}() {{ - | return {mapAccessor(node)}; - |}} - | - |/** - | * @param key the key of the value to return - | * @return {fullName} - | */ - |public {userDataType(node)} {name}(String key) {{ - | return {name}.get(key){valueAccessor(node)}; - |}} - </code>.text.stripMargin.trim - } - - def accessorForStructOrScalar(node: CNode): String = { - <code> - |/** - | * @return {node.getFullName} - | */ - |public {userDataType(node)} {node.getName}() {{ - | return {node.getName}{valueAccessor(node)}; - |}} - </code>.text.stripMargin.trim - } - - val accessors = - for { - c <- children - accessor = getAccessorCode("", c) - if (accessor.length > 0) - } yield (accessor + "\n") - accessors.mkString("\n").trim - } - - private def getStaticMethodsForInnerArray(inner: InnerCNode) = { - """ - |private static InnerNodeVector<%s> createVector(List<Builder> builders) { - | List<%s> elems = new ArrayList<>(); - | for (Builder b : builders) { - | elems.add(new %s(b)); - | } - | return new InnerNodeVector<%s>(elems); - |} - """.stripMargin.format(List.fill(5)(nodeClass(inner)): _*).trim - } - - private def getStaticMethodsForInnerMap(inner: InnerCNode) = { - """ - |private static Map<String, %s> createMap(Map<String, Builder> builders) { - | Map<String, %s> ret = new LinkedHashMap<>(); - | for(String key : builders.keySet()) { - | ret.put(key, new %s(builders.get(key))); - | } - | return Collections.unmodifiableMap(ret); - |} - """.stripMargin.format(List.fill(3)(nodeClass(inner)): _*).trim - } - - private def getEnumCode(enum: EnumLeaf, indent: String): String = { - - def getEnumValues(enum: EnumLeaf): String = { - val enumValues = - for (value <- enum.getLegalValues) yield - """ public final static Enum %s = Enum.%s;""".format(value, value) - enumValues.mkString("\n") - } - - // TODO: try to rewrite to xml - val code = - """ - |%s - |public final static class %s extends EnumNode<%s> { - - | public %s(){ - | this.value = null; - | } - - | public %s(Enum enumValue) { - | super(enumValue != null); - | this.value = enumValue; - | } - - | public enum Enum {%s} - |%s - - | @Override - | protected boolean doSetValue(@NonNull String name) { - | try { - | value = Enum.valueOf(name); - | return true; - | } catch (IllegalArgumentException e) { - | } - | return false; - | } - |} - |""" - .stripMargin.format(getClassDoc(enum, indent), - nodeClass(enum), - nodeClass(enum)+".Enum", - nodeClass(enum), - nodeClass(enum), - enum.getLegalValues.mkString(", "), - getEnumValues(enum)) - - indentCode(indent, code).trim - } - - def getClassDoc(node: CNode, indent: String): String = { - val header = "/**\n" + " * This class represents " + node.getFullName - val nodeComment = node.getCommentBlock(" *") match { - case "" => "" - case s => "\n *\n" + s.stripLineEnd // TODO: strip trailing \n in CNode.getCommentBlock - } - header + nodeComment + "\n */" - } - - def indentCode(indent: String, code: String): String = { - val indentedLines = - for (s <- code.split("\n", -1)) yield - if (s.length() > 0) (indent + s) else s - indentedLines.mkString("\n") - } - - /** - * @return the name of the class that is generated by this node. - */ - def nodeClass(node: CNode): String = { - node match { - case emptyName: CNode if node.getName.length == 0 => - throw new CodegenRuntimeException("Node with empty name, under parent " + emptyName.getParent.getName) - case root: InnerCNode if root.getParent == null => ConfiggenUtil.createClassName(root.getName) - case b: BooleanLeaf => "BooleanNode" - case d: DoubleLeaf => "DoubleNode" - case f: FileLeaf => "FileNode" - case p: PathLeaf => "PathNode" - case i: IntegerLeaf => "IntegerNode" - case l: LongLeaf => "LongNode" - case r: ReferenceLeaf => "ReferenceNode" - case s: StringLeaf => "StringNode" - case _ => node.getName.capitalize - } - } - - def userDataType(node: CNode): String = { - node match { - case inner: InnerCNode => nodeClass(node) - case enum: EnumLeaf => nodeClass(enum) + ".Enum" - case b: BooleanLeaf => "boolean" - case d: DoubleLeaf => "double" - case f: FileLeaf => "FileReference" - case p: PathLeaf => "Path" - case i: IntegerLeaf => "int" - case l: LongLeaf => "long" - case s: StringLeaf => "String" - } - } - - /** - * @return the boxed java data type, e.g. Integer for int - */ - def boxedDataType(node: CNode): String = { - val rawType = userDataType(node) - - rawType match { - case "int" => "Integer" - case _ if rawType == rawType.toLowerCase => rawType.capitalize - case _ => rawType - } - } - -} diff --git a/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala b/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala deleted file mode 100644 index e03a6d3d04b..00000000000 --- a/configgen/src/main/scala/com/yahoo/config/codegen/JavaClassBuilder.scala +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.codegen - -import java.io.{File, FileNotFoundException, FileOutputStream, PrintStream} - -import com.yahoo.config.codegen.ConfigGenerator.indentCode -import com.yahoo.config.codegen.ConfiggenUtil.createClassName -import com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX - -import scala.collection.JavaConverters._ -import scala.util.Random -/** - * Builds one Java class based on the given CNode tree. - * - * @author gjoranv - * @author tonytv - */ -class JavaClassBuilder( - root: InnerCNode, - nd: NormalizedDefinition, - destDir: File, - rawPackagePrefix: String) - extends ClassBuilder -{ - import JavaClassBuilder._ - - val packagePrefix = if (rawPackagePrefix != null) rawPackagePrefix else DEFAULT_PACKAGE_PREFIX - val javaPackage = if (root.getPackage != null) root.getPackage else packagePrefix + root.getNamespace - val className = createClassName(root.getName) - - override def createConfigClasses() { - try { - val outFile = new File(getDestPath(destDir, javaPackage), className + ".java") - var out: PrintStream = null - try { - out = new PrintStream(new FileOutputStream(outFile)) - out.print(getConfigClass(className)) - } finally { - if (out != null) out.close() - } - System.err.println(outFile.getPath + " successfully written.") - } - catch { - case e: FileNotFoundException => { - throw new CodegenRuntimeException(e) - } - } - } - - def getConfigClass(className:String): String = { - val ret = new StringBuilder - - ret.append(getHeader).append("\n\n") - ret.append(getRootClassDeclaration(root, className)).append("\n\n") - ret.append(indentCode(Indentation, getFrameworkCode(className))).append("\n\n") - ret.append(ConfigGenerator.generateContent(Indentation, root)).append("\n") - ret.append("}\n") - - ret.toString() - } - - private def getHeader: String = { - <code> - |/** - | * This file is generated from a config definition file. - | * ------------ D O N O T E D I T ! ------------ - | */ - | - |package {javaPackage}; - | - |import java.util.*; - |import java.nio.file.Path; - |import edu.umd.cs.findbugs.annotations.NonNull; - |{getImportFrameworkClasses(root.getNamespace)} - </code>.text.stripMargin.trim - } - - private def getImportFrameworkClasses(namespace: String): String = { - if (namespace != CNode.DEFAULT_NAMESPACE) - "import " + packagePrefix + CNode.DEFAULT_NAMESPACE + ".*;\n" - else - "" - } - - // TODO: remove the extra comment line " *" if root.getCommentBlock is empty - private def getRootClassDeclaration(root:InnerCNode, className: String): String = { - <code> - |/** - | * This class represents the root node of {root.getFullName} - | * - |{root.getCommentBlock(" *")} */ - |public final class {className} extends ConfigInstance {{ - | - | public final static String CONFIG_DEF_MD5 = "{root.getMd5}"; - | public final static String CONFIG_DEF_NAME = "{root.getName}"; - | public final static String CONFIG_DEF_NAMESPACE = "{root.getNamespace}"; - | public final static String CONFIG_DEF_VERSION = "{root.getVersion}"; - | public final static String[] CONFIG_DEF_SCHEMA = {{ - |{indentCode(Indentation * 2, getDefSchema)} - | }}; - | - | public static String getDefMd5() {{ return CONFIG_DEF_MD5; }} - | public static String getDefName() {{ return CONFIG_DEF_NAME; }} - | public static String getDefNamespace() {{ return CONFIG_DEF_NAMESPACE; }} - | public static String getDefVersion() {{ return CONFIG_DEF_VERSION; }} - </code>.text.stripMargin.trim - } - - private def getDefSchema: String = { - nd.getNormalizedContent.asScala.map { line => - "\"" + - line.replace("\"", "\\\"") + - "\"" - }.mkString(",\n") - } - - private def getFrameworkCode(className: String): String = { - getProducerBase - } - - private def getProducerBase = { - """ - |public interface Producer extends ConfigInstance.Producer { - | void getConfig(Builder builder); - |} - """.stripMargin.trim - } - - /** - * @param rootDir The root directory for the destination path. - * @param javaPackage The java package - * @return the destination path for the generated config file, including the given rootDir. - */ - private def getDestPath(rootDir: File, javaPackage: String): File = { - var dir: File = rootDir - val subDirs: Array[String] = javaPackage.split("""\.""") - for (subDir <- subDirs) { - dir = new File(dir, subDir) - this.synchronized { - if (!dir.isDirectory && !dir.mkdir) throw new CodegenRuntimeException("Could not create " + dir.getPath) - } - } - dir - } - -} - - -object JavaClassBuilder { - - val Indentation = " " - - /** - * Returns a name that can be safely used as a local variable in the generated config class - * for the given node. The name will be based on the given basis string, but the basis itself is - * not a possible return value. - * - * @param node The node to find a unused symbol name for. - * @param basis The basis for the generated symbol name. - * @return A name that is not used in the given config node. - */ - def createUniqueSymbol(node: CNode, basis: String) = { - - def getCandidate(cnt: Int) = { - if (cnt < basis.length()) - basis.substring(0, cnt) - else - ReservedWords.INTERNAL_PREFIX + basis + Random.nextInt().abs - } - - def getUsedSymbols: Set[String] = { - (node.getChildren map (child => child.getName)).toSet - } - - // TODO: refactoring potential - val usedSymbols = getUsedSymbols - var count = 1 - var candidate = getCandidate(count) - while (usedSymbols contains(candidate)) { - count += 1 - candidate = getCandidate(count) - } - candidate - } - -} diff --git a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java new file mode 100644 index 00000000000..744f8c9b1a2 --- /dev/null +++ b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java @@ -0,0 +1,116 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.codegen; + +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.FileSystems; +import java.nio.file.Files; + +import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; +import static com.yahoo.config.codegen.JavaClassBuilder.createUniqueSymbol; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + * @author ollivir + */ +public class JavaClassBuilderTest { + private static final String TEST_DIR = "target/test-classes/"; + private static final String DEF_NAME = TEST_DIR + "allfeatures.def"; + private static final String REFERENCE_NAME = TEST_DIR + "allfeatures.reference"; + + @Ignore + @Test + public void visual_inspection_of_generated_class() { + final String testDefinition = "version=1\n" + // + "namespace=test\n" + // + "p path\n" + // + "pathArr[] path\n" + // + "f file\n" + // + "fileArr[] file\n" + // + "i int default=0\n" + // + "# A long value\n" + // + "l long default=0\n" + // + "s string default=\"\"\n" + // + "b bool\n" + // + "# An enum value\n" + // + "e enum {A, B, C}\n" + // + "intArr[] int\n" + // + "boolArr[] bool\n" + // + "enumArr[] enum {FOO, BAR}\n" + // + "intMap{} int\n" + // + "# A struct\n" + // + "# with multi-line\n" + // + "# comment and \"quotes\".\n" + // + "myStruct.i int\n" + // + "myStruct.s string\n" + // + "# An inner array\n" + // + "myArr[].i int\n" + // + "myArr[].newStruct.s string\n" + // + "myArr[].newStruct.b bool\n" + // + "myArr[].intArr[] int\n" + // + "# An inner map\n" + // + "myMap{}.i int\n" + // + "myMap{}.newStruct.s string\n" + // + "myMap{}.newStruct.b bool\n" + // + "myMap{}.intArr[] int\n" + // + "intMap{} int\n"; + + DefParser parser = new DefParser("test", new StringReader(testDefinition)); + InnerCNode root = parser.getTree(); + JavaClassBuilder builder = new JavaClassBuilder(root, parser.getNormalizedDefinition(), null, null); + String configClass = builder.getConfigClass("TestConfig"); + System.out.print(configClass); + } + + @Test + public void testCreateUniqueSymbol() { + final String testDefinition = "version=1\n" + // + "namespace=test\n" + // + "m int\n" + // + "n int\n"; + InnerCNode root = new DefParser("test", new StringReader(testDefinition)).getTree(); + + assertThat(createUniqueSymbol(root, "foo"), is("f")); + assertThat(createUniqueSymbol(root, "name"), is("na")); + assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m")); + + // The basis string is not a legal return value, even if unique, to avoid + // multiple symbols with the same name if the same basis string is given twice. + assertTrue(createUniqueSymbol(root, "my").startsWith(ReservedWords.INTERNAL_PREFIX + "my")); + } + + @Test + public void testCreateClassName() { + assertThat(createClassName("simple"), is("SimpleConfig")); + assertThat(createClassName("a"), is("AConfig")); + assertThat(createClassName("a-b-c"), is("ABCConfig")); + assertThat(createClassName("a-1-2b"), is("A12bConfig")); + assertThat(createClassName("my-app"), is("MyAppConfig")); + assertThat(createClassName("MyApp"), is("MyAppConfig")); + } + + @Test(expected = CodegenRuntimeException.class) + public void testIllegalClassName() { + createClassName("+illegal"); + } + + @Test + public void verify_generated_class_against_reference() throws IOException { + final String testDefinition = String.join("\n", Files.readAllLines(FileSystems.getDefault().getPath(DEF_NAME))); + final String referenceClass = String.join("\n", Files.readAllLines(FileSystems.getDefault().getPath(REFERENCE_NAME))) + "\n"; + + DefParser parser = new DefParser("allfeatures", new StringReader(testDefinition)); + InnerCNode root = parser.getTree(); + JavaClassBuilder builder = new JavaClassBuilder(root, parser.getNormalizedDefinition(), null, null); + String configClass = builder.getConfigClass("AllfeaturesConfig"); + + assertEquals(referenceClass, configClass); + } +} diff --git a/configgen/src/test/resources/allfeatures.reference b/configgen/src/test/resources/allfeatures.reference new file mode 100644 index 00000000000..ebc21e8255c --- /dev/null +++ b/configgen/src/test/resources/allfeatures.reference @@ -0,0 +1,1983 @@ +/** + * This file is generated from a config definition file. + * ------------ D O N O T E D I T ! ------------ + */ + +package com.yahoo.configgen; + +import java.util.*; +import java.nio.file.Path; +import edu.umd.cs.findbugs.annotations.NonNull; +import com.yahoo.config.*; + +/** + * This class represents the root node of allfeatures + * + * Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + * + * This def file should test most aspects of def files that makes a difference + * for the generated config classes. The goal is to trigger all blocks of + * code in the code generators. This includes: + * + * - Use all legal special characters in the def file name, to ensure that those + * that needs to be replaced in type names are actually replaced. + * - Use the same enum type twice to verify that we dont declare or define it + * twice. + * - Use the same struct type twice for the same reason. + * - Include arrays of primitives and structs. + * - Include enum primitives and array of enums. Arrays of enums must be handled + * specially by the C++ code. + * - Include enums both with and without default values. + * - Include primitive string, numbers & doubles both with and without default + * values. + * - Have an array within a struct, to verify that we correctly recurse. + * - Reuse type name further within to ensure that this works. + */ +public final class AllfeaturesConfig extends ConfigInstance { + + public final static String CONFIG_DEF_MD5 = "eb2d24dbbcf054b21be729e2cfaafd93"; + public final static String CONFIG_DEF_NAME = "allfeatures"; + public final static String CONFIG_DEF_NAMESPACE = "configgen"; + public final static String CONFIG_DEF_VERSION = ""; + public final static String[] CONFIG_DEF_SCHEMA = { + "namespace=configgen", + "boolVal bool", + "bool_with_def bool default=false", + "intVal int", + "intWithDef int default=-545", + "longVal long", + "longWithDef long default=1234567890123", + "doubleVal double", + "double_with_def double default=-6.43", + "stringVal string", + "stringwithdef string default=\"foobar#notacomment\"", + "enumVal enum { FOO, BAR, FOOBAR }", + "enumwithdef enum { FOO2, BAR2, FOOBAR2 } default=BAR2", + "refVal reference", + "refwithdef reference default=\":parent:\"", + "fileVal file", + "pathVal path", + "boolarr[] bool", + "intarr[] int", + "longarr[] long", + "doublearr[] double", + "stringarr[] string", + "enumarr[] enum { ARRAY, VALUES }", + "refarr[] reference", + "filearr[] file", + "pathArr[] path", + "intMap{} int", + "pathMap{} file", + "basic_struct.foo string default=\"foo\"", + "basic_struct.bar int default=0", + "struct_of_struct.inner0.name string default=\"inner0\"", + "struct_of_struct.inner0.index int default=0", + "struct_of_struct.inner1.name string default=\"inner1\"", + "struct_of_struct.inner1.index int default=1", + "myArray[].intVal int default=14", + "myArray[].stringVal[] string", + "myArray[].enumVal enum { INNER, ENUM, TYPE } default=TYPE", + "myArray[].refVal reference", + "myArray[].anotherArray[].foo int default=-4", + "myMap{}.intVal int default=15", + "myMap{}.stringVal[] string", + "myMap{}.enumVal enum { INNER, ENUM, TYPE } default=ENUM", + "myMap{}.refVal reference", + "myMap{}.anotherArray[].foo int default=-5" + }; + + public static String getDefMd5() { return CONFIG_DEF_MD5; } + public static String getDefName() { return CONFIG_DEF_NAME; } + public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } + public static String getDefVersion() { return CONFIG_DEF_VERSION; } + + public interface Producer extends ConfigInstance.Producer { + void getConfig(Builder builder); + } + + public static class Builder implements ConfigInstance.Builder { + private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + "boolVal", + "intVal", + "longVal", + "doubleVal", + "stringVal", + "enumVal", + "refVal", + "fileVal", + "pathVal" + )); + + private Boolean boolVal = null; + private Boolean bool_with_def = null; + private Integer intVal = null; + private Integer intWithDef = null; + private Long longVal = null; + private Long longWithDef = null; + private Double doubleVal = null; + private Double double_with_def = null; + private String stringVal = null; + private String stringwithdef = null; + private EnumVal.Enum enumVal = null; + private Enumwithdef.Enum enumwithdef = null; + private String refVal = null; + private String refwithdef = null; + private String fileVal = null; + private FileReference pathVal = null; + public List<Boolean> boolarr = new ArrayList<>(); + public List<Integer> intarr = new ArrayList<>(); + public List<Long> longarr = new ArrayList<>(); + public List<Double> doublearr = new ArrayList<>(); + public List<String> stringarr = new ArrayList<>(); + public List<Enumarr.Enum> enumarr = new ArrayList<>(); + public List<String> refarr = new ArrayList<>(); + public List<String> filearr = new ArrayList<>(); + public List<FileReference> pathArr = new ArrayList<>(); + public Map<String, Integer> intMap = new LinkedHashMap<>(); + public Map<String, String> pathMap = new LinkedHashMap<>(); + public Basic_struct.Builder basic_struct = new Basic_struct.Builder(); + public Struct_of_struct.Builder struct_of_struct = new Struct_of_struct.Builder(); + public List<MyArray.Builder> myArray = new ArrayList<>(); + public Map<String, MyMap.Builder> myMap = new LinkedHashMap<>(); + + public Builder() { } + + public Builder(AllfeaturesConfig config) { + boolVal(config.boolVal()); + bool_with_def(config.bool_with_def()); + intVal(config.intVal()); + intWithDef(config.intWithDef()); + longVal(config.longVal()); + longWithDef(config.longWithDef()); + doubleVal(config.doubleVal()); + double_with_def(config.double_with_def()); + stringVal(config.stringVal()); + stringwithdef(config.stringwithdef()); + enumVal(config.enumVal()); + enumwithdef(config.enumwithdef()); + refVal(config.refVal()); + refwithdef(config.refwithdef()); + fileVal(config.fileVal().value()); + pathVal(config.pathVal.getFileReference()); + boolarr(config.boolarr()); + intarr(config.intarr()); + longarr(config.longarr()); + doublearr(config.doublearr()); + stringarr(config.stringarr()); + enumarr(config.enumarr()); + refarr(config.refarr()); + filearr(FileReference.toValues(config.filearr())); + pathArr(PathNode.toFileReferences(config.pathArr)); + intMap(config.intMap()); + pathMap(FileReference.toValueMap(config.pathMap())); + basic_struct(new Basic_struct.Builder(config.basic_struct())); + struct_of_struct(new Struct_of_struct.Builder(config.struct_of_struct())); + for (MyArray m : config.myArray()) { + myArray(new MyArray.Builder(m)); + } + for (Map.Entry<String, MyMap> __entry : config.myMap().entrySet()) { + myMap(__entry.getKey(), new MyMap.Builder(__entry.getValue())); + } + } + + private Builder override(Builder __superior) { + if (__superior.boolVal != null) + boolVal(__superior.boolVal); + if (__superior.bool_with_def != null) + bool_with_def(__superior.bool_with_def); + if (__superior.intVal != null) + intVal(__superior.intVal); + if (__superior.intWithDef != null) + intWithDef(__superior.intWithDef); + if (__superior.longVal != null) + longVal(__superior.longVal); + if (__superior.longWithDef != null) + longWithDef(__superior.longWithDef); + if (__superior.doubleVal != null) + doubleVal(__superior.doubleVal); + if (__superior.double_with_def != null) + double_with_def(__superior.double_with_def); + if (__superior.stringVal != null) + stringVal(__superior.stringVal); + if (__superior.stringwithdef != null) + stringwithdef(__superior.stringwithdef); + if (__superior.enumVal != null) + enumVal(__superior.enumVal); + if (__superior.enumwithdef != null) + enumwithdef(__superior.enumwithdef); + if (__superior.refVal != null) + refVal(__superior.refVal); + if (__superior.refwithdef != null) + refwithdef(__superior.refwithdef); + if (__superior.fileVal != null) + fileVal(__superior.fileVal); + if (__superior.pathVal != null) + pathVal(__superior.pathVal); + if (!__superior.boolarr.isEmpty()) + boolarr.addAll(__superior.boolarr); + if (!__superior.intarr.isEmpty()) + intarr.addAll(__superior.intarr); + if (!__superior.longarr.isEmpty()) + longarr.addAll(__superior.longarr); + if (!__superior.doublearr.isEmpty()) + doublearr.addAll(__superior.doublearr); + if (!__superior.stringarr.isEmpty()) + stringarr.addAll(__superior.stringarr); + if (!__superior.enumarr.isEmpty()) + enumarr.addAll(__superior.enumarr); + if (!__superior.refarr.isEmpty()) + refarr.addAll(__superior.refarr); + if (!__superior.filearr.isEmpty()) + filearr.addAll(__superior.filearr); + if (!__superior.pathArr.isEmpty()) + pathArr.addAll(__superior.pathArr); + intMap(__superior.intMap); + pathMap(__superior.pathMap); + basic_struct(basic_struct.override(__superior.basic_struct)); + struct_of_struct(struct_of_struct.override(__superior.struct_of_struct)); + if (!__superior.myArray.isEmpty()) + myArray.addAll(__superior.myArray); + myMap(__superior.myMap); + return this; + } + + public Builder boolVal(boolean __value) { + boolVal = __value; + __uninitialized.remove("boolVal"); + return this; + } + + private Builder boolVal(String __value) { + return boolVal(Boolean.valueOf(__value)); + } + + public Builder bool_with_def(boolean __value) { + bool_with_def = __value; + return this; + } + + private Builder bool_with_def(String __value) { + return bool_with_def(Boolean.valueOf(__value)); + } + + public Builder intVal(int __value) { + intVal = __value; + __uninitialized.remove("intVal"); + return this; + } + + private Builder intVal(String __value) { + return intVal(Integer.valueOf(__value)); + } + + public Builder intWithDef(int __value) { + intWithDef = __value; + return this; + } + + private Builder intWithDef(String __value) { + return intWithDef(Integer.valueOf(__value)); + } + + public Builder longVal(long __value) { + longVal = __value; + __uninitialized.remove("longVal"); + return this; + } + + private Builder longVal(String __value) { + return longVal(Long.valueOf(__value)); + } + + public Builder longWithDef(long __value) { + longWithDef = __value; + return this; + } + + private Builder longWithDef(String __value) { + return longWithDef(Long.valueOf(__value)); + } + + public Builder doubleVal(double __value) { + doubleVal = __value; + __uninitialized.remove("doubleVal"); + return this; + } + + private Builder doubleVal(String __value) { + return doubleVal(Double.valueOf(__value)); + } + + public Builder double_with_def(double __value) { + double_with_def = __value; + return this; + } + + private Builder double_with_def(String __value) { + return double_with_def(Double.valueOf(__value)); + } + + public Builder stringVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + stringVal = __value; + __uninitialized.remove("stringVal"); + return this; + } + + + public Builder stringwithdef(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + stringwithdef = __value; + return this; + } + + + public Builder enumVal(EnumVal.Enum __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + enumVal = __value; + __uninitialized.remove("enumVal"); + return this; + } + + private Builder enumVal(String __value) { + return enumVal(EnumVal.Enum.valueOf(__value)); + } + + public Builder enumwithdef(Enumwithdef.Enum __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + enumwithdef = __value; + return this; + } + + private Builder enumwithdef(String __value) { + return enumwithdef(Enumwithdef.Enum.valueOf(__value)); + } + + public Builder refVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + refVal = __value; + __uninitialized.remove("refVal"); + return this; + } + + + public Builder refwithdef(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + refwithdef = __value; + return this; + } + + + public Builder fileVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + fileVal = __value; + __uninitialized.remove("fileVal"); + return this; + } + + + public Builder pathVal(FileReference __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + pathVal = __value; + __uninitialized.remove("pathVal"); + return this; + } + + + public Builder boolarr(Boolean __value) { + boolarr.add(__value); + return this; + } + + public Builder boolarr(Collection<Boolean> __values) { + boolarr.addAll(__values); + return this; + } + + private Builder boolarr(String __value) { + return boolarr(Boolean.valueOf(__value)); + } + + public Builder intarr(Integer __value) { + intarr.add(__value); + return this; + } + + public Builder intarr(Collection<Integer> __values) { + intarr.addAll(__values); + return this; + } + + private Builder intarr(String __value) { + return intarr(Integer.valueOf(__value)); + } + + public Builder longarr(Long __value) { + longarr.add(__value); + return this; + } + + public Builder longarr(Collection<Long> __values) { + longarr.addAll(__values); + return this; + } + + private Builder longarr(String __value) { + return longarr(Long.valueOf(__value)); + } + + public Builder doublearr(Double __value) { + doublearr.add(__value); + return this; + } + + public Builder doublearr(Collection<Double> __values) { + doublearr.addAll(__values); + return this; + } + + private Builder doublearr(String __value) { + return doublearr(Double.valueOf(__value)); + } + + public Builder stringarr(String __value) { + stringarr.add(__value); + return this; + } + + public Builder stringarr(Collection<String> __values) { + stringarr.addAll(__values); + return this; + } + + public Builder enumarr(Enumarr.Enum __value) { + enumarr.add(__value); + return this; + } + + public Builder enumarr(Collection<Enumarr.Enum> __values) { + enumarr.addAll(__values); + return this; + } + + private Builder enumarr(String __value) { + return enumarr(Enumarr.Enum.valueOf(__value)); + } + + public Builder refarr(String __value) { + refarr.add(__value); + return this; + } + + public Builder refarr(Collection<String> __values) { + refarr.addAll(__values); + return this; + } + + public Builder filearr(String __value) { + filearr.add(__value); + return this; + } + + public Builder filearr(Collection<String> __values) { + filearr.addAll(__values); + return this; + } + + public Builder pathArr(FileReference __value) { + pathArr.add(__value); + return this; + } + + public Builder pathArr(Collection<FileReference> __values) { + pathArr.addAll(__values); + return this; + } + + public Builder intMap(String __key, Integer __value) { + intMap.put(__key, __value); + return this; + } + + public Builder intMap(Map<String, Integer> __values) { + intMap.putAll(__values); + return this; + } + + private Builder intMap(String __key, String __value) { + return intMap(__key, Integer.valueOf(__value)); + } + + public Builder pathMap(String __key, String __value) { + pathMap.put(__key, __value); + return this; + } + + public Builder pathMap(Map<String, String> __values) { + pathMap.putAll(__values); + return this; + } + + public Builder basic_struct(Basic_struct.Builder __builder) { + basic_struct = __builder; + return this; + } + + public Builder struct_of_struct(Struct_of_struct.Builder __builder) { + struct_of_struct = __builder; + return this; + } + + /** + * Add the given builder to this builder's list of MyArray builders + * @param __builder a builder + * @return this builder + */ + public Builder myArray(MyArray.Builder __builder) { + myArray.add(__builder); + return this; + } + + /** + * Set the given list as this builder's list of MyArray builders + * @param __builders a list of builders + * @return this builder + */ + public Builder myArray(List<MyArray.Builder> __builders) { + myArray = __builders; + return this; + } + + public Builder myMap(String __key, MyMap.Builder __value) { + myMap.put(__key, __value); + return this; + } + + public Builder myMap(Map<String, MyMap.Builder> __values) { + myMap.putAll(__values); + return this; + } + + @java.lang.Override + public final boolean dispatchGetConfig(ConfigInstance.Producer producer) { + if (producer instanceof Producer) { + ((Producer)producer).getConfig(this); + return true; + } + return false; + } + + @java.lang.Override + public final String getDefMd5() { return CONFIG_DEF_MD5; } + @java.lang.Override + public final String getDefName() { return CONFIG_DEF_NAME; } + @java.lang.Override + public final String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } + } + + // Some random bool without a default value. These comments exist to check + // that comment parsing works.e + private final BooleanNode boolVal; + // A bool with a default value set. + private final BooleanNode bool_with_def; + private final IntegerNode intVal; + private final IntegerNode intWithDef; + private final LongNode longVal; + private final LongNode longWithDef; + private final DoubleNode doubleVal; + private final DoubleNode double_with_def; + // Another comment + private final StringNode stringVal; + private final StringNode stringwithdef; + private final EnumVal enumVal; + private final Enumwithdef enumwithdef; + private final ReferenceNode refVal; + private final ReferenceNode refwithdef; + private final FileNode fileVal; + private final PathNode pathVal; + private final LeafNodeVector<Boolean, BooleanNode> boolarr; + private final LeafNodeVector<Integer, IntegerNode> intarr; + private final LeafNodeVector<Long, LongNode> longarr; + private final LeafNodeVector<Double, DoubleNode> doublearr; + private final LeafNodeVector<String, StringNode> stringarr; + private final LeafNodeVector<Enumarr.Enum, Enumarr> enumarr; + private final LeafNodeVector<String, ReferenceNode> refarr; + private final LeafNodeVector<FileReference, FileNode> filearr; + private final LeafNodeVector<Path, PathNode> pathArr; + private final Map<String, IntegerNode> intMap; + private final Map<String, FileNode> pathMap; + private final Basic_struct basic_struct; + private final Struct_of_struct struct_of_struct; + private final InnerNodeVector<MyArray> myArray; + private final Map<String, MyMap> myMap; + + public AllfeaturesConfig(Builder builder) { + this(builder, true); + } + + private AllfeaturesConfig(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures must be initialized: " + builder.__uninitialized); + + boolVal = (builder.boolVal == null) ? + new BooleanNode() : new BooleanNode(builder.boolVal); + bool_with_def = (builder.bool_with_def == null) ? + new BooleanNode(false) : new BooleanNode(builder.bool_with_def); + intVal = (builder.intVal == null) ? + new IntegerNode() : new IntegerNode(builder.intVal); + intWithDef = (builder.intWithDef == null) ? + new IntegerNode(-545) : new IntegerNode(builder.intWithDef); + longVal = (builder.longVal == null) ? + new LongNode() : new LongNode(builder.longVal); + longWithDef = (builder.longWithDef == null) ? + new LongNode(1234567890123L) : new LongNode(builder.longWithDef); + doubleVal = (builder.doubleVal == null) ? + new DoubleNode() : new DoubleNode(builder.doubleVal); + double_with_def = (builder.double_with_def == null) ? + new DoubleNode(-6.43D) : new DoubleNode(builder.double_with_def); + stringVal = (builder.stringVal == null) ? + new StringNode() : new StringNode(builder.stringVal); + stringwithdef = (builder.stringwithdef == null) ? + new StringNode("foobar#notacomment") : new StringNode(builder.stringwithdef); + enumVal = (builder.enumVal == null) ? + new EnumVal() : new EnumVal(builder.enumVal); + enumwithdef = (builder.enumwithdef == null) ? + new Enumwithdef(Enumwithdef.BAR2) : new Enumwithdef(builder.enumwithdef); + refVal = (builder.refVal == null) ? + new ReferenceNode() : new ReferenceNode(builder.refVal); + refwithdef = (builder.refwithdef == null) ? + new ReferenceNode(":parent:") : new ReferenceNode(builder.refwithdef); + fileVal = (builder.fileVal == null) ? + new FileNode() : new FileNode(builder.fileVal); + pathVal = (builder.pathVal == null) ? + new PathNode() : new PathNode(builder.pathVal); + boolarr = new LeafNodeVector<>(builder.boolarr, new BooleanNode()); + intarr = new LeafNodeVector<>(builder.intarr, new IntegerNode()); + longarr = new LeafNodeVector<>(builder.longarr, new LongNode()); + doublearr = new LeafNodeVector<>(builder.doublearr, new DoubleNode()); + stringarr = new LeafNodeVector<>(builder.stringarr, new StringNode()); + enumarr = new LeafNodeVector<>(builder.enumarr, new Enumarr()); + refarr = new LeafNodeVector<>(builder.refarr, new ReferenceNode()); + filearr = LeafNodeVector.createFileNodeVector(builder.filearr); + pathArr = LeafNodeVector.createPathNodeVector(builder.pathArr); + intMap = LeafNodeMaps.asNodeMap(builder.intMap, new IntegerNode()); + pathMap = LeafNodeMaps.asFileNodeMap(builder.pathMap); + basic_struct = new Basic_struct(builder.basic_struct, throwIfUninitialized); + struct_of_struct = new Struct_of_struct(builder.struct_of_struct, throwIfUninitialized); + myArray = MyArray.createVector(builder.myArray); + myMap = MyMap.createMap(builder.myMap); + } + + /** + * @return allfeatures.boolVal + */ + public boolean boolVal() { + return boolVal.value(); + } + + /** + * @return allfeatures.bool_with_def + */ + public boolean bool_with_def() { + return bool_with_def.value(); + } + + /** + * @return allfeatures.intVal + */ + public int intVal() { + return intVal.value(); + } + + /** + * @return allfeatures.intWithDef + */ + public int intWithDef() { + return intWithDef.value(); + } + + /** + * @return allfeatures.longVal + */ + public long longVal() { + return longVal.value(); + } + + /** + * @return allfeatures.longWithDef + */ + public long longWithDef() { + return longWithDef.value(); + } + + /** + * @return allfeatures.doubleVal + */ + public double doubleVal() { + return doubleVal.value(); + } + + /** + * @return allfeatures.double_with_def + */ + public double double_with_def() { + return double_with_def.value(); + } + + /** + * @return allfeatures.stringVal + */ + public String stringVal() { + return stringVal.value(); + } + + /** + * @return allfeatures.stringwithdef + */ + public String stringwithdef() { + return stringwithdef.value(); + } + + /** + * @return allfeatures.enumVal + */ + public EnumVal.Enum enumVal() { + return enumVal.value(); + } + + /** + * @return allfeatures.enumwithdef + */ + public Enumwithdef.Enum enumwithdef() { + return enumwithdef.value(); + } + + /** + * @return allfeatures.refVal + */ + public String refVal() { + return refVal.value(); + } + + /** + * @return allfeatures.refwithdef + */ + public String refwithdef() { + return refwithdef.value(); + } + + /** + * @return allfeatures.fileVal + */ + public FileReference fileVal() { + return fileVal.value(); + } + + /** + * @return allfeatures.pathVal + */ + public Path pathVal() { + return pathVal.value(); + } + + /** + * @return allfeatures.boolarr[] + */ + public List<Boolean> boolarr() { + return boolarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.boolarr[] + */ + public boolean boolarr(int i) { + return boolarr.get(i).value(); + } + + /** + * @return allfeatures.intarr[] + */ + public List<Integer> intarr() { + return intarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.intarr[] + */ + public int intarr(int i) { + return intarr.get(i).value(); + } + + /** + * @return allfeatures.longarr[] + */ + public List<Long> longarr() { + return longarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.longarr[] + */ + public long longarr(int i) { + return longarr.get(i).value(); + } + + /** + * @return allfeatures.doublearr[] + */ + public List<Double> doublearr() { + return doublearr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.doublearr[] + */ + public double doublearr(int i) { + return doublearr.get(i).value(); + } + + /** + * @return allfeatures.stringarr[] + */ + public List<String> stringarr() { + return stringarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.stringarr[] + */ + public String stringarr(int i) { + return stringarr.get(i).value(); + } + + /** + * @return allfeatures.enumarr[] + */ + public List<Enumarr.Enum> enumarr() { + return enumarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.enumarr[] + */ + public Enumarr.Enum enumarr(int i) { + return enumarr.get(i).value(); + } + + /** + * @return allfeatures.refarr[] + */ + public List<String> refarr() { + return refarr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.refarr[] + */ + public String refarr(int i) { + return refarr.get(i).value(); + } + + /** + * @return allfeatures.filearr[] + */ + public List<FileReference> filearr() { + return filearr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.filearr[] + */ + public FileReference filearr(int i) { + return filearr.get(i).value(); + } + + /** + * @return allfeatures.pathArr[] + */ + public List<Path> pathArr() { + return pathArr.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.pathArr[] + */ + public Path pathArr(int i) { + return pathArr.get(i).value(); + } + + /** + * @return allfeatures.intMap{} + */ + public Map<String, Integer> intMap() { + return LeafNodeMaps.asValueMap(intMap); + } + + /** + * @param key the key of the value to return + * @return allfeatures.intMap{} + */ + public int intMap(String key) { + return intMap.get(key).value(); + } + + /** + * @return allfeatures.pathMap{} + */ + public Map<String, FileReference> pathMap() { + return LeafNodeMaps.asValueMap(pathMap); + } + + /** + * @param key the key of the value to return + * @return allfeatures.pathMap{} + */ + public FileReference pathMap(String key) { + return pathMap.get(key).value(); + } + + /** + * @return allfeatures.basic_struct + */ + public Basic_struct basic_struct() { + return basic_struct; + } + + /** + * @return allfeatures.struct_of_struct + */ + public Struct_of_struct struct_of_struct() { + return struct_of_struct; + } + + /** + * @return allfeatures.myArray[] + */ + public List<MyArray> myArray() { + return myArray; + } + + /** + * @param i the index of the value to return + * @return allfeatures.myArray[] + */ + public MyArray myArray(int i) { + return myArray.get(i); + } + + /** + * @return allfeatures.myMap{} + */ + public Map<String, MyMap> myMap() { + return Collections.unmodifiableMap(myMap); + } + + /** + * @param key the key of the value to return + * @return allfeatures.myMap{} + */ + public MyMap myMap(String key) { + return myMap.get(key); + } + + private ChangesRequiringRestart getChangesRequiringRestart(AllfeaturesConfig newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("allfeatures"); + return changes; + } + + private static boolean containsFieldsFlaggedWithRestart() { + return false; + } + + /** + * This class represents allfeatures.enumVal + */ + public final static class EnumVal extends EnumNode<EnumVal.Enum> { + + public EnumVal(){ + this.value = null; + } + + public EnumVal(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {FOO, BAR, FOOBAR} + public final static Enum FOO = Enum.FOO; + public final static Enum BAR = Enum.BAR; + public final static Enum FOOBAR = Enum.FOOBAR; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.enumwithdef + */ + public final static class Enumwithdef extends EnumNode<Enumwithdef.Enum> { + + public Enumwithdef(){ + this.value = null; + } + + public Enumwithdef(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {FOO2, BAR2, FOOBAR2} + public final static Enum FOO2 = Enum.FOO2; + public final static Enum BAR2 = Enum.BAR2; + public final static Enum FOOBAR2 = Enum.FOOBAR2; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.enumarr[] + */ + public final static class Enumarr extends EnumNode<Enumarr.Enum> { + + public Enumarr(){ + this.value = null; + } + + public Enumarr(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {ARRAY, VALUES} + public final static Enum ARRAY = Enum.ARRAY; + public final static Enum VALUES = Enum.VALUES; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.basic_struct + */ + public final static class Basic_struct extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private String foo = null; + private Integer bar = null; + + public Builder() { } + + public Builder(Basic_struct config) { + foo(config.foo()); + bar(config.bar()); + } + + private Builder override(Builder __superior) { + if (__superior.foo != null) + foo(__superior.foo); + if (__superior.bar != null) + bar(__superior.bar); + return this; + } + + public Builder foo(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + foo = __value; + return this; + } + + + public Builder bar(int __value) { + bar = __value; + return this; + } + + private Builder bar(String __value) { + return bar(Integer.valueOf(__value)); + } + } + + // A basic struct + private final StringNode foo; + private final IntegerNode bar; + + public Basic_struct(Builder builder) { + this(builder, true); + } + + private Basic_struct(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.basic_struct must be initialized: " + builder.__uninitialized); + + foo = (builder.foo == null) ? + new StringNode("foo") : new StringNode(builder.foo); + bar = (builder.bar == null) ? + new IntegerNode(0) : new IntegerNode(builder.bar); + } + + /** + * @return allfeatures.basic_struct.foo + */ + public String foo() { + return foo.value(); + } + + /** + * @return allfeatures.basic_struct.bar + */ + public int bar() { + return bar.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(Basic_struct newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("basic_struct"); + return changes; + } + } + + /** + * This class represents allfeatures.struct_of_struct + */ + public final static class Struct_of_struct extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + public Inner0.Builder inner0 = new Inner0.Builder(); + public Inner1.Builder inner1 = new Inner1.Builder(); + + public Builder() { } + + public Builder(Struct_of_struct config) { + inner0(new Inner0.Builder(config.inner0())); + inner1(new Inner1.Builder(config.inner1())); + } + + private Builder override(Builder __superior) { + inner0(inner0.override(__superior.inner0)); + inner1(inner1.override(__superior.inner1)); + return this; + } + + public Builder inner0(Inner0.Builder __builder) { + inner0 = __builder; + return this; + } + + public Builder inner1(Inner1.Builder __builder) { + inner1 = __builder; + return this; + } + } + + private final Inner0 inner0; + private final Inner1 inner1; + + public Struct_of_struct(Builder builder) { + this(builder, true); + } + + private Struct_of_struct(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.struct_of_struct must be initialized: " + builder.__uninitialized); + + inner0 = new Inner0(builder.inner0, throwIfUninitialized); + inner1 = new Inner1(builder.inner1, throwIfUninitialized); + } + + /** + * @return allfeatures.struct_of_struct.inner0 + */ + public Inner0 inner0() { + return inner0; + } + + /** + * @return allfeatures.struct_of_struct.inner1 + */ + public Inner1 inner1() { + return inner1; + } + + private ChangesRequiringRestart getChangesRequiringRestart(Struct_of_struct newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("struct_of_struct"); + return changes; + } + + /** + * This class represents allfeatures.struct_of_struct.inner0 + */ + public final static class Inner0 extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private String name = null; + private Integer index = null; + + public Builder() { } + + public Builder(Inner0 config) { + name(config.name()); + index(config.index()); + } + + private Builder override(Builder __superior) { + if (__superior.name != null) + name(__superior.name); + if (__superior.index != null) + index(__superior.index); + return this; + } + + public Builder name(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + name = __value; + return this; + } + + + public Builder index(int __value) { + index = __value; + return this; + } + + private Builder index(String __value) { + return index(Integer.valueOf(__value)); + } + } + + // A struct of struct + private final StringNode name; + private final IntegerNode index; + + public Inner0(Builder builder) { + this(builder, true); + } + + private Inner0(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.struct_of_struct.inner0 must be initialized: " + builder.__uninitialized); + + name = (builder.name == null) ? + new StringNode("inner0") : new StringNode(builder.name); + index = (builder.index == null) ? + new IntegerNode(0) : new IntegerNode(builder.index); + } + + /** + * @return allfeatures.struct_of_struct.inner0.name + */ + public String name() { + return name.value(); + } + + /** + * @return allfeatures.struct_of_struct.inner0.index + */ + public int index() { + return index.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(Inner0 newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("inner0"); + return changes; + } + } + + /** + * This class represents allfeatures.struct_of_struct.inner1 + */ + public final static class Inner1 extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private String name = null; + private Integer index = null; + + public Builder() { } + + public Builder(Inner1 config) { + name(config.name()); + index(config.index()); + } + + private Builder override(Builder __superior) { + if (__superior.name != null) + name(__superior.name); + if (__superior.index != null) + index(__superior.index); + return this; + } + + public Builder name(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + name = __value; + return this; + } + + + public Builder index(int __value) { + index = __value; + return this; + } + + private Builder index(String __value) { + return index(Integer.valueOf(__value)); + } + } + + private final StringNode name; + private final IntegerNode index; + + public Inner1(Builder builder) { + this(builder, true); + } + + private Inner1(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.struct_of_struct.inner1 must be initialized: " + builder.__uninitialized); + + name = (builder.name == null) ? + new StringNode("inner1") : new StringNode(builder.name); + index = (builder.index == null) ? + new IntegerNode(1) : new IntegerNode(builder.index); + } + + /** + * @return allfeatures.struct_of_struct.inner1.name + */ + public String name() { + return name.value(); + } + + /** + * @return allfeatures.struct_of_struct.inner1.index + */ + public int index() { + return index.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(Inner1 newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("inner1"); + return changes; + } + } + } + + /** + * This class represents allfeatures.myArray[] + */ + public final static class MyArray extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + "refVal" + )); + + private Integer intVal = null; + public List<String> stringVal = new ArrayList<>(); + private EnumVal.Enum enumVal = null; + private String refVal = null; + public List<AnotherArray.Builder> anotherArray = new ArrayList<>(); + + public Builder() { } + + public Builder(MyArray config) { + intVal(config.intVal()); + stringVal(config.stringVal()); + enumVal(config.enumVal()); + refVal(config.refVal()); + for (AnotherArray a : config.anotherArray()) { + anotherArray(new AnotherArray.Builder(a)); + } + } + + private Builder override(Builder __superior) { + if (__superior.intVal != null) + intVal(__superior.intVal); + if (!__superior.stringVal.isEmpty()) + stringVal.addAll(__superior.stringVal); + if (__superior.enumVal != null) + enumVal(__superior.enumVal); + if (__superior.refVal != null) + refVal(__superior.refVal); + if (!__superior.anotherArray.isEmpty()) + anotherArray.addAll(__superior.anotherArray); + return this; + } + + public Builder intVal(int __value) { + intVal = __value; + return this; + } + + private Builder intVal(String __value) { + return intVal(Integer.valueOf(__value)); + } + + public Builder stringVal(String __value) { + stringVal.add(__value); + return this; + } + + public Builder stringVal(Collection<String> __values) { + stringVal.addAll(__values); + return this; + } + + public Builder enumVal(EnumVal.Enum __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + enumVal = __value; + return this; + } + + private Builder enumVal(String __value) { + return enumVal(EnumVal.Enum.valueOf(__value)); + } + + public Builder refVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + refVal = __value; + __uninitialized.remove("refVal"); + return this; + } + + + /** + * Add the given builder to this builder's list of AnotherArray builders + * @param __builder a builder + * @return this builder + */ + public Builder anotherArray(AnotherArray.Builder __builder) { + anotherArray.add(__builder); + return this; + } + + /** + * Set the given list as this builder's list of AnotherArray builders + * @param __builders a list of builders + * @return this builder + */ + public Builder anotherArray(List<AnotherArray.Builder> __builders) { + anotherArray = __builders; + return this; + } + } + + private final IntegerNode intVal; + private final LeafNodeVector<String, StringNode> stringVal; + private final EnumVal enumVal; + private final ReferenceNode refVal; + private final InnerNodeVector<AnotherArray> anotherArray; + + public MyArray(Builder builder) { + this(builder, true); + } + + private MyArray(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.myArray[] must be initialized: " + builder.__uninitialized); + + intVal = (builder.intVal == null) ? + new IntegerNode(14) : new IntegerNode(builder.intVal); + stringVal = new LeafNodeVector<>(builder.stringVal, new StringNode()); + enumVal = (builder.enumVal == null) ? + new EnumVal(EnumVal.TYPE) : new EnumVal(builder.enumVal); + refVal = (builder.refVal == null) ? + new ReferenceNode() : new ReferenceNode(builder.refVal); + anotherArray = AnotherArray.createVector(builder.anotherArray); + } + + /** + * @return allfeatures.myArray[].intVal + */ + public int intVal() { + return intVal.value(); + } + + /** + * @return allfeatures.myArray[].stringVal[] + */ + public List<String> stringVal() { + return stringVal.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.myArray[].stringVal[] + */ + public String stringVal(int i) { + return stringVal.get(i).value(); + } + + /** + * @return allfeatures.myArray[].enumVal + */ + public EnumVal.Enum enumVal() { + return enumVal.value(); + } + + /** + * @return allfeatures.myArray[].refVal + */ + public String refVal() { + return refVal.value(); + } + + /** + * @return allfeatures.myArray[].anotherArray[] + */ + public List<AnotherArray> anotherArray() { + return anotherArray; + } + + /** + * @param i the index of the value to return + * @return allfeatures.myArray[].anotherArray[] + */ + public AnotherArray anotherArray(int i) { + return anotherArray.get(i); + } + + private ChangesRequiringRestart getChangesRequiringRestart(MyArray newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("myArray"); + return changes; + } + + private static InnerNodeVector<MyArray> createVector(List<Builder> builders) { + List<MyArray> elems = new ArrayList<>(); + for (Builder b : builders) { + elems.add(new MyArray(b)); + } + return new InnerNodeVector<MyArray>(elems); + } + + /** + * This class represents allfeatures.myArray[].enumVal + */ + public final static class EnumVal extends EnumNode<EnumVal.Enum> { + + public EnumVal(){ + this.value = null; + } + + public EnumVal(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {INNER, ENUM, TYPE} + public final static Enum INNER = Enum.INNER; + public final static Enum ENUM = Enum.ENUM; + public final static Enum TYPE = Enum.TYPE; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.myArray[].anotherArray[] + */ + public final static class AnotherArray extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private Integer foo = null; + + public Builder() { } + + public Builder(AnotherArray config) { + foo(config.foo()); + } + + private Builder override(Builder __superior) { + if (__superior.foo != null) + foo(__superior.foo); + return this; + } + + public Builder foo(int __value) { + foo = __value; + return this; + } + + private Builder foo(String __value) { + return foo(Integer.valueOf(__value)); + } + } + + private final IntegerNode foo; + + public AnotherArray(Builder builder) { + this(builder, true); + } + + private AnotherArray(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.myArray[].anotherArray[] must be initialized: " + builder.__uninitialized); + + foo = (builder.foo == null) ? + new IntegerNode(-4) : new IntegerNode(builder.foo); + } + + /** + * @return allfeatures.myArray[].anotherArray[].foo + */ + public int foo() { + return foo.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(AnotherArray newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("anotherArray"); + return changes; + } + + private static InnerNodeVector<AnotherArray> createVector(List<Builder> builders) { + List<AnotherArray> elems = new ArrayList<>(); + for (Builder b : builders) { + elems.add(new AnotherArray(b)); + } + return new InnerNodeVector<AnotherArray>(elems); + } + } + } + + /** + * This class represents allfeatures.myMap{} + */ + public final static class MyMap extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + "refVal" + )); + + private Integer intVal = null; + public List<String> stringVal = new ArrayList<>(); + private EnumVal.Enum enumVal = null; + private String refVal = null; + public List<AnotherArray.Builder> anotherArray = new ArrayList<>(); + + public Builder() { } + + public Builder(MyMap config) { + intVal(config.intVal()); + stringVal(config.stringVal()); + enumVal(config.enumVal()); + refVal(config.refVal()); + for (AnotherArray a : config.anotherArray()) { + anotherArray(new AnotherArray.Builder(a)); + } + } + + private Builder override(Builder __superior) { + if (__superior.intVal != null) + intVal(__superior.intVal); + if (!__superior.stringVal.isEmpty()) + stringVal.addAll(__superior.stringVal); + if (__superior.enumVal != null) + enumVal(__superior.enumVal); + if (__superior.refVal != null) + refVal(__superior.refVal); + if (!__superior.anotherArray.isEmpty()) + anotherArray.addAll(__superior.anotherArray); + return this; + } + + public Builder intVal(int __value) { + intVal = __value; + return this; + } + + private Builder intVal(String __value) { + return intVal(Integer.valueOf(__value)); + } + + public Builder stringVal(String __value) { + stringVal.add(__value); + return this; + } + + public Builder stringVal(Collection<String> __values) { + stringVal.addAll(__values); + return this; + } + + public Builder enumVal(EnumVal.Enum __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + enumVal = __value; + return this; + } + + private Builder enumVal(String __value) { + return enumVal(EnumVal.Enum.valueOf(__value)); + } + + public Builder refVal(String __value) { + if (__value == null) throw new IllegalArgumentException("Null value is not allowed."); + refVal = __value; + __uninitialized.remove("refVal"); + return this; + } + + + /** + * Add the given builder to this builder's list of AnotherArray builders + * @param __builder a builder + * @return this builder + */ + public Builder anotherArray(AnotherArray.Builder __builder) { + anotherArray.add(__builder); + return this; + } + + /** + * Set the given list as this builder's list of AnotherArray builders + * @param __builders a list of builders + * @return this builder + */ + public Builder anotherArray(List<AnotherArray.Builder> __builders) { + anotherArray = __builders; + return this; + } + } + + private final IntegerNode intVal; + private final LeafNodeVector<String, StringNode> stringVal; + private final EnumVal enumVal; + private final ReferenceNode refVal; + private final InnerNodeVector<AnotherArray> anotherArray; + + public MyMap(Builder builder) { + this(builder, true); + } + + private MyMap(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.myMap{} must be initialized: " + builder.__uninitialized); + + intVal = (builder.intVal == null) ? + new IntegerNode(15) : new IntegerNode(builder.intVal); + stringVal = new LeafNodeVector<>(builder.stringVal, new StringNode()); + enumVal = (builder.enumVal == null) ? + new EnumVal(EnumVal.ENUM) : new EnumVal(builder.enumVal); + refVal = (builder.refVal == null) ? + new ReferenceNode() : new ReferenceNode(builder.refVal); + anotherArray = AnotherArray.createVector(builder.anotherArray); + } + + /** + * @return allfeatures.myMap{}.intVal + */ + public int intVal() { + return intVal.value(); + } + + /** + * @return allfeatures.myMap{}.stringVal[] + */ + public List<String> stringVal() { + return stringVal.asList(); + } + + /** + * @param i the index of the value to return + * @return allfeatures.myMap{}.stringVal[] + */ + public String stringVal(int i) { + return stringVal.get(i).value(); + } + + /** + * @return allfeatures.myMap{}.enumVal + */ + public EnumVal.Enum enumVal() { + return enumVal.value(); + } + + /** + * @return allfeatures.myMap{}.refVal + */ + public String refVal() { + return refVal.value(); + } + + /** + * @return allfeatures.myMap{}.anotherArray[] + */ + public List<AnotherArray> anotherArray() { + return anotherArray; + } + + /** + * @param i the index of the value to return + * @return allfeatures.myMap{}.anotherArray[] + */ + public AnotherArray anotherArray(int i) { + return anotherArray.get(i); + } + + private ChangesRequiringRestart getChangesRequiringRestart(MyMap newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("myMap"); + return changes; + } + + private static Map<String, MyMap> createMap(Map<String, Builder> builders) { + Map<String, MyMap> ret = new LinkedHashMap<>(); + for(String key : builders.keySet()) { + ret.put(key, new MyMap(builders.get(key))); + } + return Collections.unmodifiableMap(ret); + } + + /** + * This class represents allfeatures.myMap{}.enumVal + */ + public final static class EnumVal extends EnumNode<EnumVal.Enum> { + + public EnumVal(){ + this.value = null; + } + + public EnumVal(Enum enumValue) { + super(enumValue != null); + this.value = enumValue; + } + + public enum Enum {INNER, ENUM, TYPE} + public final static Enum INNER = Enum.INNER; + public final static Enum ENUM = Enum.ENUM; + public final static Enum TYPE = Enum.TYPE; + + @Override + protected boolean doSetValue(@NonNull String name) { + try { + value = Enum.valueOf(name); + return true; + } catch (IllegalArgumentException e) { + } + return false; + } + } + + /** + * This class represents allfeatures.myMap{}.anotherArray[] + */ + public final static class AnotherArray extends InnerNode { + + public static class Builder implements ConfigBuilder { + private Set<String> __uninitialized = new HashSet<String>(); + + private Integer foo = null; + + public Builder() { } + + public Builder(AnotherArray config) { + foo(config.foo()); + } + + private Builder override(Builder __superior) { + if (__superior.foo != null) + foo(__superior.foo); + return this; + } + + public Builder foo(int __value) { + foo = __value; + return this; + } + + private Builder foo(String __value) { + return foo(Integer.valueOf(__value)); + } + } + + private final IntegerNode foo; + + public AnotherArray(Builder builder) { + this(builder, true); + } + + private AnotherArray(Builder builder, boolean throwIfUninitialized) { + if (throwIfUninitialized && ! builder.__uninitialized.isEmpty()) + throw new IllegalArgumentException("The following builder parameters for " + + "allfeatures.myMap{}.anotherArray[] must be initialized: " + builder.__uninitialized); + + foo = (builder.foo == null) ? + new IntegerNode(-5) : new IntegerNode(builder.foo); + } + + /** + * @return allfeatures.myMap{}.anotherArray[].foo + */ + public int foo() { + return foo.value(); + } + + private ChangesRequiringRestart getChangesRequiringRestart(AnotherArray newConfig) { + ChangesRequiringRestart changes = new ChangesRequiringRestart("anotherArray"); + return changes; + } + + private static InnerNodeVector<AnotherArray> createVector(List<Builder> builders) { + List<AnotherArray> elems = new ArrayList<>(); + for (Builder b : builders) { + elems.add(new AnotherArray(b)); + } + return new InnerNodeVector<AnotherArray>(elems); + } + } + } + +} diff --git a/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala b/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala deleted file mode 100644 index c1a5eb2dd6a..00000000000 --- a/configgen/src/test/scala/com/yahoo/config/codegen/JavaClassBuilderTest.scala +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.codegen - -import org.junit.Assert.assertThat -import org.junit.Assert.assertTrue -import org.hamcrest.CoreMatchers.is -import java.io.StringReader -import ConfiggenUtil.createClassName -import JavaClassBuilder.createUniqueSymbol -import org.junit.{Ignore, Test} - -/** - * @author gjoranv - */ -class JavaClassBuilderTest { - - @Ignore - @Test - def visual_inspection_of_generated_class() { - val testDefinition = - """version=1 - |namespace=test - |p path - |pathArr[] path - |f file - |fileArr[] file - |i int default=0 - |# A long value - |l long default=0 - |s string default="" - |b bool - |# An enum value - |e enum {A, B, C} - |intArr[] int - |boolArr[] bool - |enumArr[] enum {FOO, BAR} - |intMap{} int - |# A struct - |# with multi-line - |# comment and "quotes". - |myStruct.i int - |myStruct.s string - |# An inner array - |myArr[].i int - |myArr[].newStruct.s string - |myArr[].newStruct.b bool - |myArr[].intArr[] int - |# An inner map - |myMap{}.i int - |myMap{}.newStruct.s string - |myMap{}.newStruct.b bool - |myMap{}.intArr[] int - |intMap{} int - |""".stripMargin - - val parser = new DefParser("test", new StringReader(testDefinition)) - val root = parser.getTree - val builder = new JavaClassBuilder(root, parser.getNormalizedDefinition, null, null) - val configClass = builder.getConfigClass("TestConfig") - print(configClass) - } - - @Test - def testCreateUniqueSymbol() { - val testDefinition = - """version=1 - |namespace=test - |m int - |n int - """.stripMargin - val root = new DefParser("test", new StringReader(testDefinition)).getTree - - assertThat(createUniqueSymbol(root, "foo"), is("f")) - assertThat(createUniqueSymbol(root, "name"), is("na")) - assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m")) - - // The basis string is not a legal return value, even if unique, to avoid multiple symbols - // with the same name if the same basis string is given twice. - assertTrue(createUniqueSymbol(root, "my").startsWith(ReservedWords.INTERNAL_PREFIX + "my")) - } - - @Test - def testCreateClassName() { - assertThat(createClassName("simple"), is("SimpleConfig")) - assertThat(createClassName("a"), is("AConfig")) - assertThat(createClassName("a-b-c"), is("ABCConfig")) - assertThat(createClassName("a-1-2b"), is("A12bConfig")) - assertThat(createClassName("my-app"), is("MyAppConfig")) - assertThat(createClassName("MyApp"), is("MyAppConfig")) - } - - @Test(expected=classOf[CodegenRuntimeException]) - def testIllegalClassName() { - createClassName("+illegal") - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 64bc9868034..51882cd2266 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; @@ -50,11 +51,15 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.URI; +import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -68,6 +73,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static java.nio.file.Files.readAttributes; + /** * The API for managing applications. * @@ -271,29 +278,62 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return true; } - public HttpResponse clusterControllerStatusPage(Tenant tenant, ApplicationId applicationId, String hostName, String pathSuffix) { - Application application = getApplication(tenant, applicationId); - + public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) { // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise... // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise // change the hostname and port. Exposing other paths on the cluster controller should be fine. // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components. String relativePath = "clustercontroller-status/" + pathSuffix; - return httpProxy.get(application, hostName, "container-clustercontroller", relativePath); + return httpProxy.get(getApplication(applicationId), hostName, "container-clustercontroller", relativePath); } - public Long getApplicationGeneration(Tenant tenant, ApplicationId applicationId) { - return getApplication(tenant, applicationId).getApplicationGeneration(); + public Long getApplicationGeneration(ApplicationId applicationId) { + return getApplication(applicationId).getApplicationGeneration(); } public void restart(ApplicationId applicationId, HostFilter hostFilter) { hostProvisioner.ifPresent(provisioner -> provisioner.restart(applicationId, hostFilter)); } - public HttpResponse filedistributionStatus(Tenant tenant, ApplicationId applicationId, Duration timeout) { - Application application = getApplication(tenant, applicationId); - return fileDistributionStatus.status(application, timeout); + public HttpResponse filedistributionStatus(ApplicationId applicationId, Duration timeout) { + return fileDistributionStatus.status(getApplication(applicationId), timeout); + } + + public Set<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, boolean deleteFromDisk) { + if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory"); + + // Find all file references in use + Set<String> fileReferencesInUse = new HashSet<>(); + Set<ApplicationId> applicationIds = listApplications(); + applicationIds.forEach(applicationId -> fileReferencesInUse.addAll(getApplication(applicationId).getModel().fileReferences() + .stream() + .map(FileReference::value) + .collect(Collectors.toSet()))); + log.log(LogLevel.INFO, "File references in use : " + fileReferencesInUse); + + // Find those on disk that are not in use + Set<String> fileReferencesOnDisk = new HashSet<>(); + File[] filesOnDisk = fileReferencesPath.listFiles(); + if (filesOnDisk != null) + fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); + log.log(LogLevel.INFO, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + + Instant instant = Instant.now().minus(Duration.ofDays(14)); + Set<String> fileReferencesToDelete = fileReferencesOnDisk + .stream() + .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) + .filter(fileReference -> isFileLastModifiedBefore(new File(fileReferencesPath, fileReference), instant)) + .collect(Collectors.toSet()); + if (deleteFromDisk) { + log.log(LogLevel.INFO, "Will delete file references not in use: " + fileReferencesToDelete); + fileReferencesToDelete.forEach(fileReference -> { + File file = new File(fileReferencesPath, fileReference); + if ( ! IOUtils.recursiveDeleteDir(file)) + log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath()); + }); + } + return fileReferencesToDelete; } public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) { @@ -301,22 +341,37 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode); } - private Application getApplication(Tenant tenant, ApplicationId applicationId) { + private Application getApplication(ApplicationId applicationId) { + Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); long sessionId = getSessionIdForApplication(tenant, applicationId); RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId, 0); return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty(), clock.instant()); } + private Set<ApplicationId> listApplications() { + return tenantRepository.getAllTenants().stream() + .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream()) + .collect(Collectors.toSet()); + } + + private boolean isFileLastModifiedBefore(File fileReference, Instant instant) { + BasicFileAttributes fileAttributes; + try { + fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class); + return fileAttributes.lastModifiedTime().toInstant().isBefore(instant); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + // ---------------- Convergence ---------------------------------------------------------------- - public HttpResponse serviceConvergenceCheck(Tenant tenant, ApplicationId applicationId, String hostname, URI uri) { - Application application = getApplication(tenant, applicationId); - return convergeChecker.serviceConvergenceCheck(application, hostname, uri); + public HttpResponse serviceConvergenceCheck(ApplicationId applicationId, String hostname, URI uri) { + return convergeChecker.serviceConvergenceCheck(getApplication(applicationId), hostname, uri); } - public HttpResponse serviceListToCheckForConfigConvergence(Tenant tenant, ApplicationId applicationId, URI uri) { - Application application = getApplication(tenant, applicationId); - return convergeChecker.serviceListToCheckForConfigConvergence(application, uri); + public HttpResponse serviceListToCheckForConfigConvergence(ApplicationId applicationId, URI uri) { + return convergeChecker.serviceListToCheckForConfigConvergence(getApplication(applicationId), uri); } // ---------------- Session operations ---------------------------------------------------------------- diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java index 33718774228..72a470cf937 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java @@ -17,8 +17,7 @@ import java.util.List; * Config server db is the maintainer of the serverdb directory containing def files and the file system sessions. * See also {@link com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs} which maintains directories per tenant. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class ConfigServerDB { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigServerDB.class.getName()); @@ -41,17 +40,13 @@ public class ConfigServerDB { } } - public static ConfigServerDB createTestConfigServerDb(String dbDir, String definitionsDir) { - return new ConfigServerDB(new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(dbDir) - .configDefinitionsDir(definitionsDir))); - } - // The config definitions shipped with Vespa public File classes() { return new File(Defaults.getDefaults().underVespaHome(configserverConfig.configDefinitionsDir()));} public File serverdefs() { return new File(serverDB, "serverdefs"); } + public File path() { return serverDB; } + public static void createDirectory(File d) { if (d.exists()) { if (!d.isDirectory()) { @@ -81,7 +76,4 @@ public class ConfigServerDB { } } - public ConfigserverConfig getConfigserverConfig() { - return configserverConfig; - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java index b6e8f013fd1..6828204b17c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -27,7 +27,6 @@ public interface GlobalComponentRegistry { Curator getCurator(); ConfigCurator getConfigCurator(); Metrics getMetrics(); - ConfigServerDB getServerDB(); SessionPreparer getSessionPreparer(); ConfigserverConfig getConfigserverConfig(); TenantListener getTenantListener(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java index d0b830aceaa..88f54e569df 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -31,7 +31,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry private final ConfigCurator configCurator; private final Metrics metrics; private final ModelFactoryRegistry modelFactoryRegistry; - private final ConfigServerDB serverDB; private final SessionPreparer sessionPreparer; private final RpcServer rpcServer; private final ConfigserverConfig configserverConfig; @@ -47,7 +46,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, - ConfigServerDB serverDB, SessionPreparer sessionPreparer, RpcServer rpcServer, ConfigserverConfig configserverConfig, @@ -61,7 +59,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry this.configCurator = configCurator; this.metrics = metrics; this.modelFactoryRegistry = modelFactoryRegistry; - this.serverDB = serverDB; this.sessionPreparer = sessionPreparer; this.rpcServer = rpcServer; this.configserverConfig = configserverConfig; @@ -80,8 +77,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry @Override public Metrics getMetrics() { return metrics; } @Override - public ConfigServerDB getServerDB() { return serverDB; } - @Override public SessionPreparer getSessionPreparer() { return sessionPreparer; } @Override public ConfigserverConfig getConfigserverConfig() { return configserverConfig; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java index 93cd68f6dd6..293f35558cb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -12,23 +12,24 @@ import java.io.File; /* * Holds file system directories for a tenant * - * @author tonytv + * @author Tony Vaagenes */ public class TenantFileSystemDirs { private final File serverDB; private final TenantName tenant; + public TenantFileSystemDirs(ConfigserverConfig configserverConfig, TenantName tenant) { + this(new ConfigServerDB(configserverConfig).path(), tenant); + } + + // For testing public TenantFileSystemDirs(File dir, TenantName tenant) { this.serverDB = dir; this.tenant = tenant; ConfigServerDB.createDirectory(sessionsPath()); } - public static TenantFileSystemDirs createTestDirs(TenantName tenantName) { - return new TenantFileSystemDirs(Files.createTempDir(), tenantName); - } - public File sessionsPath() { return new File(serverDB, Path.fromString("tenants").append(tenant.value()).append("sessions").getRelative()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 1c2c24cc7bb..df2287c64cb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -26,10 +26,6 @@ public class FileDirectory { private static final Logger log = Logger.getLogger(FileDirectory.class.getName()); private final File root; - public FileDirectory() { - this(FileDistribution.getDefaultFileDBPath()); - } - public FileDirectory(File rootDir) { root = rootDir; try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index 544451b8e10..2db89c2e8ed 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.jrt.ErrorCode; @@ -11,7 +12,9 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import java.io.File; import java.util.Set; import java.util.logging.Logger; @@ -22,12 +25,22 @@ public class FileDistributionImpl implements FileDistribution { private final static Logger log = Logger.getLogger(FileDistributionImpl.class.getName()); private final Supervisor supervisor = new Supervisor(new Transport()); + private final File fileReferencesDir; + + public FileDistributionImpl(ConfigserverConfig configserverConfig) { + this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); + } @Override public void startDownload(String hostName, int port, Set<FileReference> fileReferences) { startDownloadingFileReferences(hostName, port, fileReferences); } + @Override + public File getFileReferencesDir() { + return fileReferencesDir; + } + // Notifies config proxy which file references it should start downloading. It's OK if the call does not succeed, // as downloading will then start synchronously when a service requests a file reference instead private void startDownloadingFileReferences(String hostName, int port, Set<FileReference> fileReferences) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java index d6751987424..b3f3214793c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java @@ -17,8 +17,8 @@ public class FileDistributionProvider { private final FileDistribution fileDistribution; public FileDistributionProvider(File applicationDir, FileDistribution fileDistribution) { - this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory())), fileDistribution); - ensureDirExists(FileDistribution.getDefaultFileDBPath()); + this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory(fileDistribution.getFileReferencesDir()))), fileDistribution); + ensureDirExists(fileDistribution.getFileReferencesDir()); } FileDistributionProvider(FileRegistry fileRegistry, FileDistribution fileDistribution) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 001ef751e69..42bf269e9d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.config.server.filedistribution; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; @@ -17,6 +16,7 @@ import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.filedistribution.CompressedFileReference; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceData; @@ -72,7 +72,7 @@ public class FileServer { @Inject public FileServer(ConfigserverConfig configserverConfig) { - this(createConnectionPool(configserverConfig), FileDistribution.getDefaultFileDBPath()); + this(createConnectionPool(configserverConfig), new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()))); } // For testing only diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java index 728f327c829..40d75d9dbac 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java @@ -4,14 +4,23 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; +import java.io.File; import java.util.Set; /** * @author Ulf Lilleengen */ -public class MockFileDBHandler implements FileDistribution { +public class MockFileDistribution implements FileDistribution { + private final File fileReferencesDir; + + MockFileDistribution(File fileReferencesDir) { + this.fileReferencesDir = fileReferencesDir; + } @Override public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {} + @Override + public File getFileReferencesDir() { return fileReferencesDir; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java index b4ed2352d00..db70a51b2b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java @@ -4,14 +4,16 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.application.provider.MockFileRegistry; +import java.io.File; + /** * @author Ulf Lilleengen */ public class MockFileDistributionProvider extends FileDistributionProvider { public int timesCalled = 0; - public MockFileDistributionProvider() { - super(new MockFileRegistry(), new MockFileDBHandler()); + public MockFileDistributionProvider(File fileReferencesDir) { + super(new MockFileRegistry(), new MockFileDistribution(fileReferencesDir)); } public FileDistribution getFileDistribution() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 42fdb16c7ca..6bca8b1c562 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -59,13 +59,13 @@ public class ApplicationHandler extends HttpHandler { Tenant tenant = verifyTenantAndApplication(applicationId); if (isServiceConvergeRequest(request)) { - return applicationRepository.serviceConvergenceCheck(tenant, applicationId, getHostNameFromRequest(request), request.getUri()); + return applicationRepository.serviceConvergenceCheck(applicationId, getHostNameFromRequest(request), request.getUri()); } if (isClusterControllerStatusRequest(request)) { String hostName = getHostNameFromRequest(request); String pathSuffix = getPathSuffix(request); - return applicationRepository.clusterControllerStatusPage(tenant, applicationId, hostName, pathSuffix); + return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix); } if (isContentRequest(request)) { @@ -86,15 +86,15 @@ public class ApplicationHandler extends HttpHandler { } if (isServiceConvergeListRequest(request)) { - return applicationRepository.serviceListToCheckForConfigConvergence(tenant, applicationId, request.getUri()); + return applicationRepository.serviceListToCheckForConfigConvergence(applicationId, request.getUri()); } if (isFiledistributionStatusRequest(request)) { Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5)); - return applicationRepository.filedistributionStatus(tenant, applicationId, timeout); + return applicationRepository.filedistributionStatus(applicationId, timeout); } - return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(tenant, applicationId)); + return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 2c46f2968ce..a08b077699c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -3,8 +3,10 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.session.FileDistributionFactory; import com.yahoo.vespa.curator.Curator; import java.time.Duration; @@ -13,20 +15,24 @@ public class ConfigServerMaintenance extends AbstractComponent { private final TenantsMaintainer tenantsMaintainer; private final ZooKeeperDataMaintainer zooKeeperDataMaintainer; + private final FileDistributionMaintainer fileDistributionMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ConfigServerMaintenance(ConfigserverConfig configserverConfig, ApplicationRepository applicationRepository, - Curator curator) { + Curator curator, + FileDistributionFactory fileDistributionFactory) { DefaultTimes defaults = new DefaultTimes(configserverConfig); tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); - zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.zookeeperDataMaintainerInterval); + zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.defaultInterval); + fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig); } @Override public void deconstruct() { tenantsMaintainer.deconstruct(); zooKeeperDataMaintainer.deconstruct(); + fileDistributionMaintainer.deconstruct(); } /* @@ -37,7 +43,6 @@ public class ConfigServerMaintenance extends AbstractComponent { private final Duration defaultInterval; private final Duration tenantsMaintainerInterval; - private final Duration zookeeperDataMaintainerInterval; DefaultTimes(ConfigserverConfig configserverConfig) { boolean isCd = configserverConfig.system().equals(SystemName.cd.name()); @@ -45,7 +50,6 @@ public class ConfigServerMaintenance extends AbstractComponent { this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); // TODO: Want job control or feature flag to control when to run this, for now use a very long interval unless in CD this.tenantsMaintainerInterval = isCd ? defaultInterval : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); - this.zookeeperDataMaintainerInterval = defaultInterval; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java new file mode 100644 index 00000000000..58141a3a045 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.time.Duration; + +public class FileDistributionMaintainer extends Maintainer { + + private final ApplicationRepository applicationRepository; + private final File fileReferencesDir; + + public FileDistributionMaintainer(ApplicationRepository applicationRepository, + Curator curator, + Duration interval, + ConfigserverConfig configserverConfig) { + super(applicationRepository, curator, interval); + this.applicationRepository = applicationRepository; + this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));; + } + + + @Override + protected void maintain() { + // TODO: Does not delete, for now just outputs what should be deleted + applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, false); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java index 852768b6937..fd57732eb30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java @@ -19,5 +19,6 @@ public class ZooKeeperDataMaintainer extends Maintainer { @Override protected void maintain() { curator.delete(Path.fromString("/vespa/filedistribution")); + curator.delete(Path.fromString("/vespa/config")); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 551987f4dca..9d6247c5f80 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.config.server.provision.StaticProvisioner; import java.net.URI; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -146,7 +147,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { List<MODELRESULT> allApplicationVersions = new ArrayList<>(); allApplicationVersions.add(latestModelVersion); - if (zone().environment() == Environment.dev) + if (Arrays.asList(Environment.dev, Environment.test).contains(zone().environment())) versions = keepThoseUsedOn(allocatedHosts.get(), versions); // TODO: We use the allocated hosts from the newest version when building older model versions. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java index d3a74486d12..8394494adca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.vespa.config.server.filedistribution.FileDistributionImpl; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; @@ -14,8 +16,15 @@ import java.io.File; @SuppressWarnings("WeakerAccess") public class FileDistributionFactory { + private final ConfigserverConfig configserverConfig; + + @Inject + public FileDistributionFactory(ConfigserverConfig configserverConfig) { + this.configserverConfig = configserverConfig; + } + public FileDistributionProvider createProvider(File applicationPackage) { - return new FileDistributionProvider(applicationPackage, new FileDistributionImpl()); + return new FileDistributionProvider(applicationPackage, new FileDistributionImpl(configserverConfig)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 69721ed01d4..ad967f49964 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -14,9 +14,7 @@ import com.yahoo.vespa.config.server.application.ZKTenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.session.*; -import com.yahoo.vespa.defaults.Defaults; -import java.io.File; import java.time.Clock; import java.util.Collections; @@ -162,12 +160,7 @@ public class TenantBuilder { private void createServerDbDirs() { if (tenantFileSystemDirs == null) { - tenantFileSystemDirs = new TenantFileSystemDirs(new File( - Defaults.getDefaults().underVespaHome(componentRegistry - .getServerDB() - .getConfigserverConfig() - .configServerDBDir())), - tenant); + tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigserverConfig(), tenant); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 058e39eea9b..f7e76c2b3bb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -10,10 +10,9 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest; +import com.yahoo.io.IOUtils; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -21,14 +20,17 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Collections; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -48,18 +50,22 @@ public class ApplicationRepositoryTest { private Tenant tenant; private ApplicationRepository applicationRepository; + private TenantRepository tenantRepository; private TimeoutBudget timeoutBudget; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setup() { Curator curator = new MockCurator(); - TenantRepository tenants = new TenantRepository(new TestComponentRegistry.Builder() + tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() .curator(curator) .build()); - tenants.addTenant(tenantName); - tenant = tenants.getTenant(tenantName); + tenantRepository.addTenant(tenantName); + tenant = tenantRepository.getTenant(tenantName); Provisioner provisioner = new SessionHandlerTest.MockProvisioner(); - applicationRepository = new ApplicationRepository(tenants, provisioner, clock); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); } @@ -79,15 +85,15 @@ public class ApplicationRepositoryTest { } @Test - public void createAndPrepareAndActivate() throws IOException { - PrepareResult result = deployApp(); + public void createAndPrepareAndActivate() { + PrepareResult result = deployApp(testApp); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); assertTrue(result.configChangeActions().getRestartActions().isEmpty()); } @Test - public void deleteUnusedTenants() throws IOException { - deployApp(); + public void deleteUnusedTenants() { + deployApp(testApp); assertTrue(applicationRepository.removeUnusedTenants().isEmpty()); applicationRepository.remove(applicationId()); assertEquals(tenantName, applicationRepository.removeUnusedTenants().iterator().next()); @@ -110,17 +116,58 @@ public class ApplicationRepositoryTest { assertEquals(Vtag.currentVersion, ApplicationRepository.decideVersion(regularApp, Environment.perf, targetVersion)); } + @Test + public void deleteUnusedFileReferences() throws IOException { + File fileReferencesDir = temporaryFolder.newFolder(); + + // Add file reference that is not in use and should be deleted (older than 14 days) + File filereferenceDir = createFilereferenceOnDisk(new File(fileReferencesDir, "foo"), Instant.now().minus(Duration.ofDays(15))); + // Add file reference that is not in use, but should not be deleted (not older than 14 days) + File filereferenceDir2 = createFilereferenceOnDisk(new File(fileReferencesDir, "baz"), Instant.now()); + + tenantRepository.addTenant(tenantName); + tenant = tenantRepository.getTenant(tenantName); + Provisioner provisioner = new SessionHandlerTest.MockProvisioner(); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock); + timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); + + // TODO: Deploy an app with a bundle or file that will be a file reference, too much missing in test setup to get this working now + PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build(); + deployApp(new File("src/test/apps/app"), prepareParams); + + boolean deleteFiles = false; + Set<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles); + assertEquals(Collections.singleton("foo"), toBeDeleted); + assertTrue(filereferenceDir.exists()); + assertTrue(filereferenceDir2.exists()); + + deleteFiles = true; + toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles); + assertEquals(Collections.singleton("foo"), toBeDeleted); + assertFalse(filereferenceDir.exists()); + assertTrue(filereferenceDir2.exists()); + } + + private File createFilereferenceOnDisk(File filereferenceDir, Instant lastModifiedTime) { + assertTrue(filereferenceDir.mkdir()); + File bar = new File(filereferenceDir, "file"); + IOUtils.writeFile(bar, Utf8.toBytes("test")); + assertTrue(filereferenceDir.setLastModified(lastModifiedTime.toEpochMilli())); + return filereferenceDir; + } + private PrepareResult prepareAndActivateApp(File application) throws IOException { FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application); long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, appDir.getAppDir()); return applicationRepository.prepareAndActivate(tenant, sessionId, prepareParams(), false, false, Instant.now()); } - private PrepareResult deployApp() throws IOException { - File file = CompressedApplicationInputStreamTest.createTarFile(); - return applicationRepository.deploy(CompressedApplicationInputStream.createFromCompressedStream( - new FileInputStream(file), ApplicationApiHandler.APPLICATION_X_GZIP), - prepareParams(), false, false, Instant.now()); + private PrepareResult deployApp(File applicationPackage) { + return deployApp(applicationPackage, prepareParams()); + } + + private PrepareResult deployApp(File applicationPackage, PrepareParams prepareParams) { + return applicationRepository.deploy(applicationPackage, prepareParams); } private PrepareParams prepareParams() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 77550b3cd35..67cc87ae223 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.container.jdisc.state.StateMonitor; @@ -9,12 +8,12 @@ import com.yahoo.jdisc.core.SystemTimer; import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.IOException; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; @@ -35,7 +34,7 @@ public class ConfigServerBootstrapTest { @Test public void testBootStrap() throws Exception { - ConfigserverConfig configserverConfig = createConfigserverConfig(); + ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig); tester.deployApp("myApp", "4.5.6", Instant.now()); @@ -55,7 +54,7 @@ public class ConfigServerBootstrapTest { @Test public void testBootStrapWhenRedeploymentFails() throws Exception { - ConfigserverConfig configserverConfig = createConfigserverConfig(); + ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig); tester.deployApp("myApp", "4.5.6", Instant.now()); @@ -90,8 +89,8 @@ public class ConfigServerBootstrapTest { throw new RuntimeException(messageIfWaitingFails); } - private MockRpc createRpcServer(ConfigserverConfig configserverConfig) { - return new MockRpc(configserverConfig.rpcport()); + private MockRpc createRpcServer(ConfigserverConfig configserverConfig) throws IOException { + return new MockRpc(configserverConfig.rpcport(), temporaryFolder.newFolder()); } private StateMonitor createStateMonitor() { @@ -99,10 +98,10 @@ public class ConfigServerBootstrapTest { new SystemTimer()); } - private static ConfigserverConfig createConfigserverConfig() { + private static ConfigserverConfig createConfigserverConfig(TemporaryFolder temporaryFolder) throws IOException { return new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir().getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir().getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) .hostedVespa(true) .multitenant(true)); } @@ -111,8 +110,8 @@ public class ConfigServerBootstrapTest { volatile boolean isRunning = false; - MockRpc(int port) { - super(port); + MockRpc(int port, File tempDir) { + super(port, tempDir); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java index c34f6512f29..0c8a9e5c3d8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.io.IOUtils; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -13,25 +15,28 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class ConfigServerDBTest { private ConfigServerDB serverDB; - private File dbDir; - private File definitionsDir; + private ConfigserverConfig configserverConfig; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void setup() { - dbDir = Files.createTempDir(); - definitionsDir = Files.createTempDir(); - serverDB = ConfigServerDB.createTestConfigServerDb(dbDir.getAbsolutePath(), definitionsDir.getAbsolutePath()); + public void setup() throws IOException { + configserverConfig = new ConfigserverConfig( + new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath())); + serverDB = new ConfigServerDB(configserverConfig); } private void createInitializer() throws IOException { File existingDef = new File(serverDB.classes(), "test.def"); IOUtils.writeFile(existingDef, "hello", false); - ConfigServerDB.createTestConfigServerDb(dbDir.getAbsolutePath(), definitionsDir.getAbsolutePath()); + new ConfigServerDB(configserverConfig); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java index dec9dd991de..5ca3deab1fe 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java @@ -1,11 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.filedistribution.FileServer; @@ -22,8 +20,11 @@ import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.model.VespaModelFactory; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.util.Collections; import static org.hamcrest.Matchers.is; @@ -31,14 +32,12 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class InjectedGlobalComponentRegistryTest { private Curator curator; private Metrics metrics; - private ConfigServerDB serverDB; private SessionPreparer sessionPreparer; private ConfigserverConfig configserverConfig; private RpcServer rpcServer; @@ -50,33 +49,36 @@ public class InjectedGlobalComponentRegistryTest { private ModelFactoryRegistry modelFactoryRegistry; private Zone zone; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupRegistry() { + public void setupRegistry() throws IOException { curator = new MockCurator(); ConfigCurator configCurator = ConfigCurator.create(curator); metrics = Metrics.createTestMetrics(); modelFactoryRegistry = new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))); configserverConfig = new ConfigserverConfig( new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir().getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir().getAbsolutePath())); - serverDB = new ConfigServerDB(configserverConfig); + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath())); sessionPreparer = new SessionTest.MockSessionPreparer(); - rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath())); + rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(temporaryFolder.newFolder("filereferences"))); generationCounter = new SuperModelGenerationCounter(curator); defRepo = new StaticConfigDefinitionRepo(); permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig); hostRegistries = new HostRegistries(); HostProvisionerProvider hostProvisionerProvider = HostProvisionerProvider.withProvisioner(new SessionHandlerTest.MockProvisioner()); zone = Zone.defaultZone(); - globalComponentRegistry = new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, serverDB, sessionPreparer, rpcServer, configserverConfig, generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone); + globalComponentRegistry = + new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, sessionPreparer, rpcServer, configserverConfig, + generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone); } @Test public void testThatAllComponentsAreSetup() { assertThat(globalComponentRegistry.getModelFactoryRegistry(), is(modelFactoryRegistry)); - assertThat(globalComponentRegistry.getServerDB(), is(serverDB)); assertThat(globalComponentRegistry.getSessionPreparer(), is(sessionPreparer)); assertThat(globalComponentRegistry.getMetrics(), is(metrics)); assertThat(globalComponentRegistry.getCurator(), is(curator)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index d026989a43e..e4e45d3a014 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.model.VespaModelFactory; +import java.io.File; import java.time.Clock; import java.util.Collections; import java.util.Optional; @@ -37,7 +38,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final Curator curator; private final ConfigCurator configCurator; private final Metrics metrics; - private final ConfigServerDB serverDB; private final SessionPreparer sessionPreparer; private final ConfigserverConfig configserverConfig; private final SuperModelGenerationCounter superModelGenerationCounter; @@ -57,7 +57,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { PermanentApplicationPackage permanentApplicationPackage, FileDistributionFactory fileDistributionFactory, SuperModelGenerationCounter superModelGenerationCounter, - ConfigServerDB configServerDB, HostRegistries hostRegistries, ConfigserverConfig configserverConfig, SessionPreparer sessionPreparer, @@ -71,7 +70,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.configCurator = configCurator; this.metrics = metrics; this.configserverConfig = configserverConfig; - this.serverDB = configServerDB; this.reloadListener = reloadListener; this.tenantListener = tenantListener; this.superModelGenerationCounter = superModelGenerationCounter; @@ -155,7 +153,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); FileDistributionFactory fileDistributionFactory = this.fileDistributionFactory - .orElse(new MockFileDistributionFactory()); + .orElse(new MockFileDistributionFactory(new File(configserverConfig.fileReferencesDir()))); HostProvisionerProvider hostProvisionerProvider = hostProvisioner.isPresent() ? HostProvisionerProvider.withProvisioner(hostProvisioner.get()) : HostProvisionerProvider.empty(); @@ -168,7 +166,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { permApp, fileDistributionFactory, new SuperModelGenerationCounter(curator), - new ConfigServerDB(configserverConfig), hostRegistries, configserverConfig, sessionPreparer, hostProvisioner, defRepo, reloadListener, tenantListener, zone, clock); @@ -182,8 +179,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { @Override public Metrics getMetrics() { return metrics; } @Override - public ConfigServerDB getServerDB() { return serverDB; } - @Override public SessionPreparer getSessionPreparer() { return sessionPreparer; } @Override public ConfigserverConfig getConfigserverConfig() { return configserverConfig; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 157c36d7aef..9ba8adf0aa4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -6,7 +6,9 @@ import com.yahoo.io.IOUtils; import com.yahoo.net.HostName; import com.yahoo.vespa.filedistribution.FileReferenceData; import org.junit.After; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -35,6 +37,9 @@ public class FileServerTest { created.add(dir); } + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test public void requireThatExistingFileCanBeFound() throws IOException { createCleanDir("123"); @@ -71,10 +76,12 @@ public class FileServerTest { } @Test - public void requireThatDifferentNumberOfConfigServersWork() { + public void requireThatDifferentNumberOfConfigServersWork() throws IOException { // Empty connection pool in tests etc. - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - FileServer fileServer = new FileServer(new ConfigserverConfig(builder)); + ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()); + FileServer fileServer = createFileServer(builder); assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); // Empty connection pool when only one server, no use in downloading from yourself @@ -84,7 +91,7 @@ public class FileServerTest { serverBuilder.port(123456); servers.add(serverBuilder); builder.zookeeperserver(servers); - fileServer = new FileServer(new ConfigserverConfig(builder)); + fileServer = createFileServer(builder); assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); // connection pool of size 1 when 2 servers @@ -93,10 +100,16 @@ public class FileServerTest { serverBuilder2.port(123456); servers.add(serverBuilder2); builder.zookeeperserver(servers); - fileServer = new FileServer(new ConfigserverConfig(builder)); + fileServer = createFileServer(builder); assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); } + private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException { + File fileReferencesDir = temporaryFolder.newFolder(); + configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath()); + return new FileServer(new ConfigserverConfig(configBuilder)); + } + private static class FileReceiver implements FileServer.Receiver { CompletableFuture<byte []> content; FileReceiver(CompletableFuture<byte []> content) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 2a05688c435..fba4e40000d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -50,7 +50,9 @@ import com.yahoo.vespa.model.VespaModelFactory; import org.hamcrest.core.Is; import org.junit.Before; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayOutputStream; import java.io.File; @@ -89,6 +91,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { private TenantRepository tenantRepository; private SessionActiveHandler handler; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setup() { remoteSessionRepo = new RemoteSessionRepo(tenantName); @@ -221,9 +226,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { return session; } - private void addLocalSession(long sessionId, DeployData deployData, SessionZooKeeperClient zkc) { + private void addLocalSession(long sessionId, DeployData deployData, SessionZooKeeperClient zkc) throws IOException { writeApplicationId(zkc, deployData.getApplicationName()); - TenantFileSystemDirs tenantFileSystemDirs = TenantFileSystemDirs.createTestDirs(tenantName); + TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(temporaryFolder.newFolder(), tenantName); ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(testApp, deployData); localRepo.addSession(new LocalSession(tenantName, sessionId, new SessionTest.MockSessionPreparer(), new SessionContext(app, zkc, new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java index 1c886adde73..ceb9b7129b4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java @@ -20,11 +20,13 @@ public class ZooKeeperDataMaintainerTest { curator.create(Path.fromString("/foo")); curator.create(Path.fromString("/vespa/bar")); curator.create(Path.fromString("/vespa/filedistribution")); + curator.create(Path.fromString("/vespa/config")); new ZooKeeperDataMaintainer(tester.applicationRepository(), curator, Duration.ofDays(1)).run(); assertTrue(curator.exists(Path.fromString("/foo"))); assertTrue(curator.exists(Path.fromString("/vespa"))); assertFalse(curator.exists(Path.fromString("/vespa/filedistribution"))); + assertFalse(curator.exists(Path.fromString("/vespa/config"))); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java index 0126a9e2f29..9455798c4ea 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java @@ -12,8 +12,11 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; import com.yahoo.vespa.config.server.GetConfigContext; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -28,10 +31,13 @@ import static org.junit.Assert.assertTrue; */ public class DelayedConfigResponseTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test - public void testDelayedConfigResponses() { + public void testDelayedConfigResponses() throws IOException { - MockRpc rpc = new MockRpc(13337); + MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder()); DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); assertThat(responses.size(), is(0)); JRTServerConfigRequest req = createRequest("foo", "md5", "myid", "mymd5", 3, 1000000, "bar"); @@ -50,9 +56,9 @@ public class DelayedConfigResponseTest { } @Test - public void testDelayResponseRemove() { + public void testDelayResponseRemove() throws IOException { GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId()); - MockRpc rpc = new MockRpc(13337); + MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder()); DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); responses.delayResponse(createRequest("foolio", "md5", "myid", "mymd5", 3, 100000, "bar"), context); assertThat(responses.size(), is(1)); @@ -61,8 +67,8 @@ public class DelayedConfigResponseTest { } @Test - public void testDelayedConfigResponse() { - MockRpc rpc = new MockRpc(13337); + public void testDelayedConfigResponse() throws IOException { + MockRpc rpc = new MockRpc(13337, temporaryFolder.newFolder()); DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); assertThat(responses.size(), is(0)); assertThat(responses.toString(), is("DelayedConfigResponses. Average Size=0")); @@ -72,7 +78,7 @@ public class DelayedConfigResponseTest { assertThat(rpc.latestRequest, is(req)); } - public JRTServerConfigRequest createRequest(String configName, String defMd5, String configId, String md5, long generation, long timeout, String namespace) { + private JRTServerConfigRequest createRequest(String configName, String defMd5, String configId, String md5, long generation, long timeout, String namespace) { Request request = JRTClientConfigRequestV3. createWithParams(new ConfigKey<>(configName, configId, namespace, defMd5, null), DefContent.fromList(Collections.emptyList()), "fromHost", md5, generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java index 50c39b74fa2..1a4d04d0323 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessorTest.java @@ -16,10 +16,11 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; -import com.yahoo.vespa.config.server.rpc.GetConfigProcessor; -import com.yahoo.vespa.config.server.rpc.MockRpc; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + import static org.junit.Assert.assertFalse; import java.io.IOException; @@ -39,9 +40,12 @@ import static org.junit.Assert.assertTrue; */ public class GetConfigProcessorTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test - public void testSentinelConfig() { - MockRpc rpc = new MockRpc(13337, false); + public void testSentinelConfig() throws IOException { + MockRpc rpc = new MockRpc(13337, false, temporaryFolder.newFolder()); rpc.response = new MockConfigResponse("foo"); // should be a sentinel config, but it does not matter for this test // one tenant, which has host1 assigned diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index 5a9735f774a..b4de201bd0b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Version; import com.yahoo.vespa.config.protocol.ConfigResponse; @@ -14,6 +13,7 @@ import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; +import java.io.File; import java.util.Optional; import java.util.concurrent.CompletionService; @@ -36,20 +36,20 @@ public class MockRpc extends RpcServer { public volatile JRTServerConfigRequest latestRequest = null; - public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication) { + public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication, File tempDir) { super(createConfig(port), null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath())); + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(tempDir)); if (createDefaultTenant) { onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication)); } } - public MockRpc(int port, boolean createDefaultTenant) { - this(port, createDefaultTenant, true); + public MockRpc(int port, boolean createDefaultTenant, File tempDir) { + this(port, createDefaultTenant, true, tempDir); } - public MockRpc(int port) { - this(port, true); + public MockRpc(int port, File tempDir) { + this(port, true, tempDir); } /** Reset fields used to assert on the calls made to this */ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 6bb566963c0..86412f97c20 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -48,7 +48,7 @@ public class RpcServerTest extends TestWithRpc { testEmptyConfigHostedVespa(); } - private void testEmptyConfigHostedVespa() throws InterruptedException { + private void testEmptyConfigHostedVespa() throws InterruptedException, IOException { rpcServer.onTenantDelete(TenantName.defaultName()); rpcServer.onTenantsLoaded(); JRTClientConfigRequest clientReq = createSimpleRequest(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java index e022b622fb0..845e7c0f914 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.TenantName; import com.yahoo.jrt.Request; @@ -20,7 +19,10 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; import org.junit.After; import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -36,8 +38,7 @@ import static org.junit.Assert.assertTrue; /** * Test running rpc server. * - * @author lulf - * @since 5.17 + * @author Ulf Lilleengen */ // TODO: Make this a Tester instead of a superclass public class TestWithRpc { @@ -56,8 +57,11 @@ public class TestWithRpc { private List<Integer> allocatedPorts; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupRpc() throws InterruptedException { + public void setupRpc() throws InterruptedException, IOException { allocatedPorts = new ArrayList<>(); port = allocatePort(); spec = createSpec(port); @@ -80,7 +84,7 @@ public class TestWithRpc { return port; } - protected void createAndStartRpcServer(boolean hostedVespa) { + protected void createAndStartRpcServer(boolean hostedVespa) throws IOException { ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder() .rpcport(port) @@ -94,7 +98,7 @@ public class TestWithRpc { emptyNodeFlavors(), generationCounter)), Metrics.createTestMetrics(), new HostRegistries(), - hostLivenessTracker, new FileServer(FileDistribution.getDefaultFileDBPath())); + hostLivenessTracker, new FileServer(temporaryFolder.newFolder())); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); t = new Thread(rpcServer); t.start(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index 73caf770512..987dd8a6c4d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -14,7 +14,9 @@ import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.time.Duration; @@ -34,6 +36,9 @@ public class LocalSessionRepoTest extends TestWithCurator { private ManualClock clock; private static final TenantName tenantName = TenantName.defaultName(); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setupSessions() throws Exception { setupSessions(tenantName, true); @@ -41,7 +46,7 @@ public class LocalSessionRepoTest extends TestWithCurator { private void setupSessions(TenantName tenantName, boolean createInitialSessions) throws Exception { GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - TenantFileSystemDirs tenantFileSystemDirs = TenantFileSystemDirs.createTestDirs(tenantName); + TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(temporaryFolder.newFolder(), tenantName); if (createInitialSessions) { IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.sessionsPath(), "1")); IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.sessionsPath(), "2")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index bd5cfdf3a0e..316c439a3cd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.*; import com.yahoo.path.Path; @@ -29,8 +30,7 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; /** - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class LocalSessionTest { @@ -41,7 +41,7 @@ public class LocalSessionTest { private SuperModelGenerationCounter superModelGenerationCounter; @Before - public void setupTest() throws Exception { + public void setupTest() { curator = new MockCurator(); configCurator = ConfigCurator.create(curator); superModelGenerationCounter = new SuperModelGenerationCounter(curator); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java index 9d8b7c5cc00..2c05017d449 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionProvider; @@ -11,7 +12,12 @@ import java.io.File; */ public class MockFileDistributionFactory extends FileDistributionFactory { - public final MockFileDistributionProvider mockFileDistributionProvider = new MockFileDistributionProvider(); + public final MockFileDistributionProvider mockFileDistributionProvider; + + public MockFileDistributionFactory(File fileReferencesDir) { + super(new ConfigserverConfig(new ConfigserverConfig.Builder())); + mockFileDistributionProvider = new MockFileDistributionProvider(fileReferencesDir); + } @Override public FileDistributionProvider createProvider(File applicationFile) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index a5f83021bee..7d68e7b6679 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -96,4 +96,7 @@ public abstract class DocsumField { */ public abstract Object convert(Inspector value); + /** Returns whether this is the string field type. */ + boolean isString() { return false; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index c517742f0e5..1160ea0a204 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -228,6 +228,13 @@ public class FastHit extends Hit { } @Override + public void forEachFieldAsRaw(RawUtf8Consumer consumer) { + super.forEachField(consumer); + for (SummaryData summaryData : summaries) + summaryData.forEachFieldAsRaw(consumer); + } + + @Override public Map<String, Object> fields() { Map<String, Object> fields = new HashMap<>(); for (Iterator<Map.Entry<String, Object>> i = fieldIterator(); i.hasNext(); ) { @@ -534,9 +541,28 @@ public class FastHit extends Hit { void forEachField(BiConsumer<String, Object> consumer) { data.traverse((ObjectTraverser)(name, value) -> { - Object convertedValue = type.convert(name, value); - if ( convertedValue != null && !shadowed(name) && !removed(name)) { - consumer.accept(name, convertedValue); + if (!shadowed(name) && !removed(name)) { + Object convertedValue = type.convert(name, value); + if (convertedValue != null) + consumer.accept(name, convertedValue); + } + }); + } + + void forEachFieldAsRaw(RawUtf8Consumer consumer) { + data.traverse((ObjectTraverser)(name, value) -> { + if (!shadowed(name) && !removed(name)) { + DocsumField fieldType = type.getField(name); + if (fieldType != null) { + if (fieldType.isString()) { + byte[] utf8Value = value.asUtf8(); + consumer.accept(name, utf8Value, 0, utf8Value.length); + } else { + Object convertedValue = fieldType.convert(value); + if (convertedValue != null) + consumer.accept(name, convertedValue); + } + } } }); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 54db3b083d7..30e2adab182 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -244,7 +244,7 @@ public class FastSearcher extends VespaBackEndSearcher { if (result.isFilled(summaryClass)) return; Query query = result.getQuery(); - traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass)); + traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 1, quotedSummaryClass(summaryClass)); if (query.properties().getBoolean(dispatchSummaries, true) && ! summaryNeedsQuery(query) diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java index 0fa4b7ee342..408cbbbb62d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java @@ -31,4 +31,6 @@ public class StringField extends DocsumField { return value.asString(""); } + boolean isString() { return true; } + } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 6c7018317c3..55c846ccb5b 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -68,6 +68,7 @@ import java.util.function.LongSupplier; * JSON renderer for search results. * * @author Steinar Knutsen + * @author bratseth */ // NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc. public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @@ -75,18 +76,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug"); private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback"); - private enum RenderDecision { - YES, NO, DO_NOT_KNOW; - - boolean booleanValue() { - switch (this) { - case YES: return true; - case NO: return false; - default: throw new IllegalStateException(); - } - } - } - // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString private static final String BUCKET_LIMITS = "limits"; private static final String BUCKET_TO = "to"; @@ -133,6 +122,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private final JsonFactory generatorFactory; private JsonGenerator generator; + private FieldConsumer fieldConsumer; private Deque<Integer> renderedChildren; private boolean debugRendering; private LongSupplier timeSource; @@ -304,9 +294,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void init() { super.init(); - generator = null; - renderedChildren = null; debugRendering = false; + setGenerator(null, debugRendering); + renderedChildren = null; timeSource = System::currentTimeMillis; stream = null; } @@ -314,9 +304,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void beginResponse(OutputStream stream) throws IOException { beginJsonCallback(stream); - generator = generatorFactory.createGenerator(stream, JsonEncoding.UTF8); - renderedChildren = new ArrayDeque<>(); debugRendering = getDebugRendering(getResult().getQuery()); + setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering); + renderedChildren = new ArrayDeque<>(); generator.writeStartObject(); renderTrace(getExecution().trace()); renderTiming(); @@ -473,17 +463,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return ! (hit instanceof DefaultErrorHit); } - private void fieldsStart(MutableBoolean hasFieldsField) throws IOException { - if (hasFieldsField.get()) return; - generator.writeObjectFieldStart(FIELDS); - hasFieldsField.set(true); - } - - private void fieldsEnd(MutableBoolean hasFieldsField) throws IOException { - if ( ! hasFieldsField.get()) return; - generator.writeEndObject(); - } - private void renderHitContents(Hit hit) throws IOException { String id = hit.getDisplayId(); if (id != null) @@ -509,39 +488,14 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } private void renderAllFields(Hit hit) throws IOException { - MutableBoolean hasFieldsField = new MutableBoolean(false); - renderTotalHitCount(hit, hasFieldsField); - renderStandardFields(hit, hasFieldsField); - fieldsEnd(hasFieldsField); - } - - private void renderStandardFields(Hit hit, MutableBoolean hasFieldsField) { - hit.forEachField((name, value) -> { - try { - if (shouldRender(name, value)) { - fieldsStart(hasFieldsField); - renderField(name, value); - } - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - }); + fieldConsumer.startHitFields(); + renderTotalHitCount(hit); + renderStandardFields(hit); + fieldConsumer.endHitFields(); } - private boolean shouldRender(String name, Object value) { - if (debugRendering) return true; - - if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; - - if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false; - - // StringFieldValue cannot hold a null, so checking length directly is OK: - if (value instanceof StringFieldValue && ((StringFieldValue) value).getString().isEmpty()) return false; - - if (value instanceof NanNumber) return false; - - return true; + private void renderStandardFields(Hit hit) { + hit.forEachFieldAsRaw(fieldConsumer); } private void renderSpecialCasesForGrouping(Hit hit) throws IOException { @@ -606,96 +560,13 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return (id instanceof RawBucketId ? Arrays.toString(((RawBucketId) id).getTo()) : id.getTo()).toString(); } - private void renderTotalHitCount(Hit hit, MutableBoolean hasFieldsField) throws IOException { + private void renderTotalHitCount(Hit hit) throws IOException { if ( ! (getRecursionLevel() == 1 && hit instanceof HitGroup)) return; - fieldsStart(hasFieldsField); + fieldConsumer.ensureFieldsField(); generator.writeNumberField(TOTAL_COUNT, getResult().getTotalHitCount()); - } - - private void renderField(String name, Object value) throws IOException { - generator.writeFieldName(name); - renderFieldContents(value); - } - - private void renderFieldContents(Object field) throws IOException { - if (field == null) { - generator.writeNull(); - } else if (field instanceof Number) { - renderNumberField((Number) field); - } else if (field instanceof TreeNode) { - generator.writeTree((TreeNode) field); - } else if (field instanceof Tensor) { - renderTensor(Optional.of((Tensor)field)); - } else if (field instanceof JsonProducer) { - generator.writeRawValue(((JsonProducer) field).toJson()); - } else if (field instanceof Inspectable) { - StringBuilder intermediate = new StringBuilder(); - JsonRender.render((Inspectable) field, intermediate, true); - generator.writeRawValue(intermediate.toString()); - } else if (field instanceof StringFieldValue) { - // This needs special casing as JsonWriter hides empty strings now - generator.writeString(((StringFieldValue)field).getString()); - } else if (field instanceof TensorFieldValue) { - renderTensor(((TensorFieldValue)field).getTensor()); - } else if (field instanceof FieldValue) { - // the null below is the field which has already been written - ((FieldValue) field).serialize(null, new JsonWriter(generator)); - } else if (field instanceof JSONArray || field instanceof JSONObject) { - // org.json returns null if the object would not result in - // syntactically correct JSON - String s = field.toString(); - if (s == null) { - generator.writeNull(); - } else { - generator.writeRawValue(s); - } - } else { - generator.writeString(field.toString()); - } - } - - private void renderNumberField(Number field) throws IOException { - if (field instanceof Integer) { - generator.writeNumber(field.intValue()); - } else if (field instanceof Float) { - generator.writeNumber(field.floatValue()); - } else if (field instanceof Double) { - generator.writeNumber(field.doubleValue()); - } else if (field instanceof Long) { - generator.writeNumber(field.longValue()); - } else if (field instanceof Byte || field instanceof Short) { - generator.writeNumber(field.intValue()); - } else if (field instanceof BigInteger) { - generator.writeNumber((BigInteger) field); - } else if (field instanceof BigDecimal) { - generator.writeNumber((BigDecimal) field); - } else { - generator.writeNumber(field.doubleValue()); - } - } - - private void renderTensor(Optional<Tensor> tensor) throws IOException { - generator.writeStartObject(); - generator.writeArrayFieldStart("cells"); - if (tensor.isPresent()) { - for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) { - Tensor.Cell cell = i.next(); - - generator.writeStartObject(); - - generator.writeObjectFieldStart("address"); - for (int d = 0; d < cell.getKey().size(); d++) - generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d)); - generator.writeEndObject(); - - generator.writeObjectField("value", cell.getValue()); - - generator.writeEndObject(); - } - } - generator.writeEndArray(); - generator.writeEndObject(); + // alternative for the above two lines: + // fieldConsumer.accept(TOTAL_COUNT, getResult().getTotalHitCount()); } @Override @@ -774,11 +645,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return null; } - /** - * Only for testing. Never to be used in any other context. - */ - void setGenerator(JsonGenerator generator) { + private void setGenerator(JsonGenerator generator, boolean debugRendering) { this.generator = generator; + this.fieldConsumer = generator == null ? null : new FieldConsumer(generator, debugRendering); } /** @@ -787,5 +656,170 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { void setTimeSource(LongSupplier timeSource) { this.timeSource = timeSource; } - + + /** + * Received callbacks when fields of hits are encountered. + * This instance is reused for all hits of a Result since we are in a single-threaded context + * and want to limit object creation. + */ + private static class FieldConsumer implements Hit.RawUtf8Consumer { + + private final JsonGenerator generator; + private final boolean debugRendering; + + private MutableBoolean hasFieldsField; + + public FieldConsumer(JsonGenerator generator, boolean debugRendering) { + this.generator = generator; + this.debugRendering = debugRendering; + } + + /** + * Call before using this for a hit to track whether we + * have created the "fields" field of the JSON object + */ + void startHitFields() { + this.hasFieldsField = new MutableBoolean(false); + } + + /** Call before rendering a field to the generator */ + void ensureFieldsField() throws IOException { + if (hasFieldsField.get()) return; + generator.writeObjectFieldStart(FIELDS); + hasFieldsField.set(true); + } + + /** Call after all fields in a hit to close the "fields" field of the JSON object */ + void endHitFields() throws IOException { + if ( ! hasFieldsField.get()) return; + generator.writeEndObject(); + this.hasFieldsField = null; + } + + @Override + public void accept(String name, Object value) { + try { + if (shouldRender(name, value)) { + ensureFieldsField(); + generator.writeFieldName(name); + renderFieldContents(value); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void accept(String name, byte[] utf8Data, int offset, int length) { + try { + if (shouldRenderUtf8Value(name, length)) { + ensureFieldsField(); + generator.writeFieldName(name); + generator.writeUTF8String(utf8Data, offset, length); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private boolean shouldRender(String name, Object value) { + if (debugRendering) return true; + if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; + if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false; + // StringFieldValue cannot hold a null, so checking length directly is OK: + if (value instanceof StringFieldValue && ((StringFieldValue) value).getString().isEmpty()) return false; + if (value instanceof NanNumber) return false; + return true; + } + + private boolean shouldRenderUtf8Value(String name, int length) { + if (debugRendering) return true; + if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; + if (length == 0) return false; + return true; + } + + private void renderFieldContents(Object field) throws IOException { + if (field == null) { + generator.writeNull(); + } else if (field instanceof Number) { + renderNumberField((Number) field); + } else if (field instanceof TreeNode) { + generator.writeTree((TreeNode) field); + } else if (field instanceof Tensor) { + renderTensor(Optional.of((Tensor)field)); + } else if (field instanceof JsonProducer) { + generator.writeRawValue(((JsonProducer) field).toJson()); + } else if (field instanceof Inspectable) { + StringBuilder intermediate = new StringBuilder(); + JsonRender.render((Inspectable) field, intermediate, true); + generator.writeRawValue(intermediate.toString()); + } else if (field instanceof StringFieldValue) { + generator.writeString(((StringFieldValue)field).getString()); + } else if (field instanceof TensorFieldValue) { + renderTensor(((TensorFieldValue)field).getTensor()); + } else if (field instanceof FieldValue) { + // the null below is the field which has already been written + ((FieldValue) field).serialize(null, new JsonWriter(generator)); + } else if (field instanceof JSONArray || field instanceof JSONObject) { + // org.json returns null if the object would not result in + // syntactically correct JSON + String s = field.toString(); + if (s == null) { + generator.writeNull(); + } else { + generator.writeRawValue(s); + } + } else { + generator.writeString(field.toString()); + } + } + + private void renderNumberField(Number field) throws IOException { + if (field instanceof Integer) { + generator.writeNumber(field.intValue()); + } else if (field instanceof Float) { + generator.writeNumber(field.floatValue()); + } else if (field instanceof Double) { + generator.writeNumber(field.doubleValue()); + } else if (field instanceof Long) { + generator.writeNumber(field.longValue()); + } else if (field instanceof Byte || field instanceof Short) { + generator.writeNumber(field.intValue()); + } else if (field instanceof BigInteger) { + generator.writeNumber((BigInteger) field); + } else if (field instanceof BigDecimal) { + generator.writeNumber((BigDecimal) field); + } else { + generator.writeNumber(field.doubleValue()); + } + } + + private void renderTensor(Optional<Tensor> tensor) throws IOException { + generator.writeStartObject(); + generator.writeArrayFieldStart("cells"); + if (tensor.isPresent()) { + for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) { + Tensor.Cell cell = i.next(); + + generator.writeStartObject(); + + generator.writeObjectFieldStart("address"); + for (int d = 0; d < cell.getKey().size(); d++) + generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d)); + generator.writeEndObject(); + + generator.writeObjectField("value", cell.getValue()); + + generator.writeEndObject(); + } + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index f68916c8a68..74c31aa33c5 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -404,13 +404,25 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** * Receive a callback on the given object for each field in this hit. - * This is the most resource efficient way of traversing all the fields of a hit. + * This is more efficient than accessing the fields as a map or iterator. */ public void forEachField(BiConsumer<String, Object> consumer) { if (fields == null) return; fields.forEach(consumer); } + /** + * Receive a callback on the given object for each field in this hit, + * where the callback will provide raw utf-8 byte data for strings whose data + * is already available at this form. + * This is the most resource efficient way of traversing all the fields of a hit + * in renderers which produces utf-8. + */ + public void forEachFieldAsRaw(RawUtf8Consumer consumer) { + if (fields == null) return; + fields.forEach(consumer); // No utf-8 fields available in Hit + } + /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ public Map<String, Object> fields() { return getUnmodifiableFieldMap(); } @@ -800,4 +812,18 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return "hit " + getId() + " (relevance " + getRelevance() + ")"; } + public interface RawUtf8Consumer extends BiConsumer<String, Object> { + + /** + * Called for fields which are available as UTF-8 instead of accept(String, Object). + * + * @param fieldName the name of the field + * @param utf8Data raw utf-8 data. The reciver <b>must not</b> modify this data + * @param offset the start index of the data to accept into the utf8Data array + * @param length the length of the data to accept into the utf8Data array + */ + void accept(String fieldName, byte[] utf8Data, int offset, int length); + + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index a9eb9c5e6ce..421f70c8d07 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -141,9 +141,9 @@ public class SlimeSummaryTestCase { @Test public void testFieldAccessAPI() { - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf); DocsumDefinitionSet partialDocsum2 = createDocsumDefinitionSet(partial_summary2_cf); + DocsumDefinitionSet fullDocsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); Map<String, Object> expected = new HashMap<>(); @@ -261,6 +261,18 @@ public class SlimeSummaryTestCase { fieldIterator.remove(); expected.remove("integer_field"); assertFields(expected, hit); + + // --- Add full summary + assertNull(fullDocsum.lazyDecode("default", fullishSummary(), hit)); + expected.put("integer_field", 4); + expected.put("short_field", (short)2); + expected.put("byte_field", (byte)1); + expected.put("float_field", 4.5f); + expected.put("double_field", 8.75d); + expected.put("int64_field", 8L); + expected.put("string_field", "string_value"); + expected.put("longstring_field", "longstring_value"); + assertFields(expected, hit); } @@ -274,11 +286,16 @@ public class SlimeSummaryTestCase { traversed.put(name, value); }); assertEquals(expected, traversed); + // raw utf8 field traverser + Map<String, Object> traversedUtf8 = new HashMap<>(); + hit.forEachFieldAsRaw(new Utf8FieldTraverser(traversedUtf8)); + assertEquals(expected, traversedUtf8); // fieldKeys int fieldNameIteratorFieldCount = 0; for (Iterator<String> i = hit.fieldKeys().iterator(); i.hasNext(); ) { fieldNameIteratorFieldCount++; - assertTrue(expected.containsKey(i.next())); + String name = i.next(); + assertTrue("Expected field " + name, expected.containsKey(name)); } assertEquals(expected.size(), fieldNameIteratorFieldCount); // fieldKeys @@ -304,7 +321,7 @@ public class SlimeSummaryTestCase { return encode((slime)); } - private byte [] timeoutSummary() { + private byte[] timeoutSummary() { Slime slime = new Slime(); slime.setString("Timed out...."); return encode((slime)); @@ -327,6 +344,22 @@ public class SlimeSummaryTestCase { return encode((slime)); } + private byte[] fullishSummary() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setLong("integer_field", 4); + docsum.setLong("short_field", 2); + docsum.setLong("byte_field", 1); + docsum.setDouble("float_field", 4.5); + docsum.setDouble("double_field", 8.75); + docsum.setLong("int64_field", 8); + docsum.setString("string_field", "string_value"); + //docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8)); + docsum.setString("longstring_field", "longstring_value"); + //docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8)); + return encode((slime)); + } + private byte[] fullSummary(Tensor tensor1, Tensor tensor2) { Slime slime = new Slime(); Cursor docsum = slime.setObject(); @@ -346,8 +379,10 @@ public class SlimeSummaryTestCase { field.setLong("foo", 1); field.setLong("bar", 2); } - docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1)); - docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2)); + if (tensor1 != null) + docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1)); + if (tensor2 != null) + docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2)); return encode((slime)); } @@ -371,4 +406,26 @@ public class SlimeSummaryTestCase { return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); } + private static class Utf8FieldTraverser implements Hit.RawUtf8Consumer { + + private Map<String, Object> traversed; + + public Utf8FieldTraverser(Map<String, Object> traversed) { + this.traversed = traversed; + } + + @Override + public void accept(String fieldName, byte[] utf8Data, int offset, int length) { + traversed.put(fieldName, new String(utf8Data, offset, length, StandardCharsets.UTF_8)); + } + + @Override + public void accept(String name, Object value) { + if (name.equals("string_value")) + fail("Expected string_value to be received as UTF-8"); + traversed.put(name, value); + } + + } + } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 4a0075c0cf1..bf56ad19f44 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -101,7 +101,7 @@ public class JsonRendererTestCase { } @Test - public void testDocumentId() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testDocumentId() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index a2b420c8090..b15acd726d7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; -import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -38,8 +37,6 @@ public interface ConfigServer { void deactivate(DeploymentId applicationInstance) throws NoInstanceException; - JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds); - ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 8fb37ba2a15..b65d3bc0849 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.fasterxml.jackson.databind.JsonNode; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; @@ -10,7 +9,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; @@ -157,11 +155,6 @@ public class Controller extends AbstractComponent { return globalRoutingService.getHealthStatus(rotation.name()); } - // TODO: Model the response properly - public JsonNode waitForConfigConvergence(DeploymentId deploymentId, long timeout) { - return configServer.waitForConfigConverge(deploymentId, timeout); - } - public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) { return configServer.getApplicationView(tenantName, applicationName, instanceName, environment, region); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 516cd52d710..ac0d08f5105 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -38,12 +37,8 @@ public class SystemUpgrader extends Maintainer { if (!target.isPresent()) { return; } - // TODO: Change to SystemApplication.all() once host applications support upgrade - try { - deploy(Arrays.asList(SystemApplication.configServer, SystemApplication.zone), target.get()); - } catch (Exception e) { - log.log(Level.WARNING, "Failed to upgrade system. Retrying in " + maintenanceInterval(), e); - } + + deploy(SystemApplication.all(), target.get()); } /** Deploy a list of system applications until they converge on the given version */ @@ -51,14 +46,14 @@ public class SystemUpgrader extends Maintainer { for (List<ZoneId> zones : controller().zoneRegistry().upgradePolicy().asList()) { boolean converged = true; for (ZoneId zone : zones) { - for (SystemApplication application : applications) { - boolean dependenciesConverged = application.dependencies().stream() - .filter(applications::contains) // TODO: Remove when all() is used. - .allMatch(dependency -> currentVersion(zone, dependency.id()).equals(target)); - if (dependenciesConverged) { - deploy(target, application, zone); - } - converged &= currentVersion(zone, application.id()).equals(target); + try { + converged &= deployInZone(zone, applications, target); + } catch (UnreachableNodeRepositoryException e) { + converged = false; + log.log(Level.WARNING, e.getMessage() + ". Continuing to next parallel deployed zone"); + } catch (Exception e) { + converged = false; + log.log(Level.WARNING, "Failed to upgrade " + zone + ". Continuing to next parallel deployed zone", e); } } if (!converged) { @@ -67,6 +62,18 @@ public class SystemUpgrader extends Maintainer { } } + /** @return true if all applications have converged to the target version in the zone */ + private boolean deployInZone(ZoneId zone, List<SystemApplication> applications, Version target) { + boolean converged = true; + for (SystemApplication application : applications) { + if (convergedOn(target, application.dependencies(), zone)) { + deploy(target, application, zone); + } + converged &= convergedOn(target, application, zone); + } + return converged; + } + /** Deploy application on given version idempotently */ private void deploy(Version target, SystemApplication application, ZoneId zone) { if (!wantedVersion(zone, application.id(), target).equals(target)) { @@ -75,12 +82,20 @@ public class SystemUpgrader extends Maintainer { } } + private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneId zone) { + return applications.stream().allMatch(application -> convergedOn(target, application, zone)); + } + + private boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { + return currentVersion(zone, application.id(), target).equals(target); + } + private Version wantedVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { return minVersion(zone, application, Node::wantedVersion).orElse(defaultVersion); } - private Version currentVersion(ZoneId zone, ApplicationId application) { - return minVersion(zone, application, Node::currentVersion).orElse(Version.emptyVersion); + private Version currentVersion(ZoneId zone, ApplicationId application, Version defaultVersion) { + return minVersion(zone, application, Node::currentVersion).orElse(defaultVersion); } private Optional<Version> minVersion(ZoneId zone, ApplicationId application, Function<Node, Version> versionField) { @@ -92,9 +107,8 @@ public class SystemUpgrader extends Maintainer { .map(versionField) .min(Comparator.naturalOrder()); } catch (Exception e) { - log.log(Level.WARNING, String.format("Failed to get version for %s in %s: %s", application, zone, - Exceptions.toMessageString(e))); - return Optional.empty(); + throw new UnreachableNodeRepositoryException(String.format("Failed to get version for %s in %s: %s", + application, zone, Exceptions.toMessageString(e))); } } @@ -105,4 +119,9 @@ public class SystemUpgrader extends Maintainer { .map(VespaVersion::versionNumber); } + private class UnreachableNodeRepositoryException extends RuntimeException { + private UnreachableNodeRepositoryException(String reason) { + super(reason); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d9848577f0a..10088ba3fea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -168,7 +168,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/converge")) return waitForConvergence(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -579,12 +578,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse waitForConvergence(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - return new JacksonJsonResponse(controller.waitForConfigConvergence(new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - ZoneId.from(environment, region)), - asLong(request.getProperty("timeout"), 1000))); - } - private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java deleted file mode 100644 index cfd6feccf01..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.container.jdisc.HttpResponse; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * @author bratseth - */ -public class JacksonJsonResponse extends HttpResponse { - - private final JsonNode node; - - public JacksonJsonResponse(JsonNode node) { - super(200); - this.node = node; - } - - @Override - public void render(OutputStream stream) throws IOException { - new ObjectMapper().writeValue(stream, node); - } - - @Override - public String getContentType() { return "application/json"; } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java index 99b913709df..ac9afd9752a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java @@ -1,9 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; @@ -186,13 +183,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer applications.remove(deployment.applicationId()); } - @Override - public JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds) { - ObjectNode root = new ObjectNode(JsonNodeFactory.instance); - root.put("generation", 1); - return root; - } - // Returns a canned example response @Override public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 1ec64f8d478..4b563ed203d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -121,7 +121,6 @@ public class SystemUpgraderTest { } @Test - @Ignore // TODO: Unignore once host applications support upgrade public void upgrade_system_containing_host_applications() { tester.controllerTester().zoneRegistry().setUpgradePolicy( UpgradePolicy.create() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 407263a7973..8d734ec549c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -294,14 +293,12 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + // POST a 'restart application' command with a host filter (other filters not supported yet) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1", POST) .screwdriverIdentity(SCREWDRIVER_ID), "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); - // GET (wait for) convergence - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", GET) - .userIdentity(USER_ID), - new File("convergence.json")); + // GET services tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", GET) .userIdentity(USER_ID), @@ -319,11 +316,13 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", DELETE) .userIdentity(USER_ID), "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"); + // DELETE (deactivate) a deployment - prod tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + // DELETE (deactivate) a deployment is idempotent tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json deleted file mode 100644 index acfb67b702b..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "generation": 1 -}
\ No newline at end of file diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index b6aca1ec8be..97dd916bc60 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -200,8 +200,6 @@ DocumentUpdateTest::testSimpleUsage() { ByteBuffer::UP docBuf = serialize42(docUpdate); docBuf->flip(); DocumentUpdate::UP docUpdateCopy(DocumentUpdate::create42(repo, *docBuf)); - CPPUNIT_ASSERT(!docUpdate.affectsDocumentBody()); - CPPUNIT_ASSERT(!docUpdateCopy->affectsDocumentBody()); // Create a test document Document doc(*docType, DocumentId("doc::testdoc")); @@ -252,7 +250,6 @@ DocumentUpdateTest::testSimpleUsage() { updated.getValue("intarr").release())); CPPUNIT_ASSERT_EQUAL(size_t(3), val->size()); CPPUNIT_ASSERT_EQUAL(4, (*val)[2].getAsInt()); - CPPUNIT_ASSERT(upd.affectsDocumentBody()); } { Document updated(doc); @@ -593,24 +590,21 @@ void DocumentUpdateTest::testReadSerializedFile() DocumentUpdate& upd(*updp); const DocumentType *type = repo.getDocumentType("serializetest"); - CPPUNIT_ASSERT_EQUAL(DocumentId(DocIdString("update", "test")), - upd.getId()); + CPPUNIT_ASSERT_EQUAL(DocumentId(DocIdString("update", "test")), upd.getId()); CPPUNIT_ASSERT_EQUAL(*type, upd.getType()); // Verify assign value update. - FieldUpdate serField = upd[0]; + FieldUpdate serField = upd.getUpdates()[0]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("intfield").getId()); const ValueUpdate* serValue = &serField[0]; CPPUNIT_ASSERT_EQUAL(serValue->getType(), ValueUpdate::Assign); - const AssignValueUpdate* assign( - static_cast<const AssignValueUpdate*>(serValue)); - CPPUNIT_ASSERT_EQUAL(IntFieldValue(4), - static_cast<const IntFieldValue&>(assign->getValue())); + const AssignValueUpdate* assign(static_cast<const AssignValueUpdate*>(serValue)); + CPPUNIT_ASSERT_EQUAL(IntFieldValue(4), static_cast<const IntFieldValue&>(assign->getValue())); // Verify clear field update. - serField = upd[1]; + serField = upd.getUpdates()[1]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("floatfield").getId()); serValue = &serField[0]; @@ -618,7 +612,7 @@ void DocumentUpdateTest::testReadSerializedFile() CPPUNIT_ASSERT(serValue->inherits(ClearValueUpdate::classId)); // Verify add value update. - serField = upd[2]; + serField = upd.getUpdates()[2]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("arrayoffloatfield").getId()); serValue = &serField[0]; @@ -765,14 +759,11 @@ DocumentUpdateTest::testUpdateArrayEmptyParamValue() TestDocMan docMan; Document::UP doc(docMan.createDocument()); const Field &field(doc->getType().getField("tags")); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, - doc->getValue(field).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign array field with no array values = empty array. DocumentUpdate update(*doc->getDataType(), doc->getId()); - update.addUpdate(FieldUpdate(field) - .addUpdate(AssignValueUpdate( - ArrayFieldValue(field.getDataType())))); + update.addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(ArrayFieldValue(field.getDataType())))); update.applyTo(*doc); // Verify that the field was set in the document. @@ -781,10 +772,9 @@ DocumentUpdateTest::testUpdateArrayEmptyParamValue() CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size()); // Remove array field. - update.clear(); - update.addUpdate(FieldUpdate(field) - .addUpdate(ClearValueUpdate())); - update.applyTo(*doc); + DocumentUpdate update2(*doc->getDataType(), doc->getId()); + update2.addUpdate(FieldUpdate(field).addUpdate(ClearValueUpdate())); + update2.applyTo(*doc); // Verify that the field was cleared in the document. std::unique_ptr<ArrayFieldValue> fval2(doc->getAs<ArrayFieldValue>(field)); @@ -798,31 +788,25 @@ DocumentUpdateTest::testUpdateWeightedSetEmptyParamValue() TestDocMan docMan; Document::UP doc(docMan.createDocument()); const Field &field(doc->getType().getField("stringweightedset")); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, - doc->getValue(field).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign weighted set with no items = empty set. DocumentUpdate update(*doc->getDataType(), doc->getId()); - update.addUpdate(FieldUpdate(field) - .addUpdate(AssignValueUpdate( - WeightedSetFieldValue(field.getDataType())))); + update.addUpdate(FieldUpdate(field).addUpdate(AssignValueUpdate(WeightedSetFieldValue(field.getDataType())))); update.applyTo(*doc); // Verify that the field was set in the document. - std::unique_ptr<WeightedSetFieldValue> - fval1(doc->getAs<WeightedSetFieldValue>(field)); + auto fval1(doc->getAs<WeightedSetFieldValue>(field)); CPPUNIT_ASSERT(fval1.get()); CPPUNIT_ASSERT_EQUAL((size_t) 0, fval1->size()); // Remove weighted set field. - update.clear(); - update.addUpdate(FieldUpdate(field) - .addUpdate(ClearValueUpdate())); - update.applyTo(*doc); + DocumentUpdate update2(*doc->getDataType(), doc->getId()); + update2.addUpdate(FieldUpdate(field).addUpdate(ClearValueUpdate())); + update2.applyTo(*doc); // Verify that the field was cleared in the document. - std::unique_ptr<WeightedSetFieldValue> - fval2(doc->getAs<WeightedSetFieldValue>(field)); + auto fval2(doc->getAs<WeightedSetFieldValue>(field)); CPPUNIT_ASSERT(!fval2); } @@ -833,8 +817,7 @@ DocumentUpdateTest::testUpdateArrayWrongSubtype() TestDocMan docMan; Document::UP doc(docMan.createDocument()); const Field &field(doc->getType().getField("tags")); - CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, - doc->getValue(field).get()); + CPPUNIT_ASSERT_EQUAL((document::FieldValue*) 0, doc->getValue(field).get()); // Assign int values to string array. DocumentUpdate update(*doc->getDataType(), doc->getId()); diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 89f9c0dab67..7058af95828 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -221,12 +221,10 @@ ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { try{ - bool affectsBody = a.affectsDocumentBody(); ByteBuffer::UP bb(serializeHEAD(a)); bb->flip(); DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, *bb)); - CPPUNIT_ASSERT_EQUAL(affectsBody, b->affectsDocumentBody()); CPPUNIT_ASSERT_EQUAL(size_t(0), bb->getRemaining()); CPPUNIT_ASSERT_EQUAL(a.getId().toString(), b->getId().toString()); CPPUNIT_ASSERT_EQUAL(a.getUpdates().size(), b->getUpdates().size()); @@ -1021,13 +1019,11 @@ FieldPathUpdateTestCase::testAffectsDocumentBody() // structmap is body field { DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{janitor}", std::string(), fv4)); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); docUp.addFieldPathUpdate(update1); - CPPUNIT_ASSERT(docUp.affectsDocumentBody()); } // strfoo is header field @@ -1037,7 +1033,6 @@ FieldPathUpdateTestCase::testAffectsDocumentBody() "strfoo", std::string(), StringFieldValue("helloworld"))); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); docUp.addFieldPathUpdate(update1); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); } } @@ -1070,7 +1065,6 @@ FieldPathUpdateTestCase::testSerializeAssign() val.setValue("rating", IntFieldValue(100)); DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); FieldPathUpdate::CP update1(new AssignFieldPathUpdate(*doc->getDataType(), "structmap{ribbit}", "true", val)); static_cast<AssignFieldPathUpdate&>(*update1).setCreateMissingPath(true); @@ -1091,7 +1085,6 @@ FieldPathUpdateTestCase::testSerializeAdd() adds.add(StringFieldValue("george is getting upset!")); DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); FieldPathUpdate::CP update1(new AddFieldPathUpdate(*doc->getDataType(), "strarray", std::string(), adds)); docUp.addFieldPathUpdate(update1); @@ -1106,7 +1099,6 @@ FieldPathUpdateTestCase::testSerializeRemove() MapFieldValue mfv(doc->getType().getField("structmap").getDataType()); DocumentUpdate docUp(_foobar_type, DocumentId("doc:barbar:foofoo")); - CPPUNIT_ASSERT(!docUp.affectsDocumentBody()); FieldPathUpdate::CP update1(new RemoveFieldPathUpdate("structmap{ribbit}", std::string())); docUp.addFieldPathUpdate(update1); diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.h b/document/src/vespa/document/serialization/vespadocumentserializer.h index 3b0fe581c9e..818759d35b5 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentserializer.h @@ -76,7 +76,7 @@ private: void write(const AssignFieldPathUpdate &value); void write(const RemoveFieldPathUpdate &value); - void visit(const DocumentUpdate &value) override { write42(value); } + void visit(const DocumentUpdate &value) override { writeHEAD(value); } void visit(const FieldUpdate &value) override { write(value); } void visit(const RemoveValueUpdate &value) override { write(value); } void visit(const AddValueUpdate &value) override { write(value); } diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index fa289ab0bfa..a3cc2a95155 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -21,27 +21,14 @@ using namespace vespalib::xml; namespace document { -IMPLEMENT_IDENTIFIABLE(DocumentUpdate, vespalib::Identifiable); - // Declare content bits. static const unsigned char CONTENT_HASTYPE = 0x01; -typedef std::vector<FieldUpdate> FieldUpdateList; -typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateList; - -DocumentUpdate::DocumentUpdate() - : _documentId("doc::"), - _type(DataType::DOCUMENT), - _updates(), - _version(Document::getNewestSerializationVersion()), - _createIfNonExistent(false) -{ -} - DocumentUpdate::DocumentUpdate(const DataType &type, const DocumentId& id) : _documentId(id), _type(&type), _updates(), + _fieldPathUpdates(), _version(Document::getNewestSerializationVersion()), _createIfNonExistent(false) { @@ -72,7 +59,7 @@ DocumentUpdate::DocumentUpdate(const DocumentTypeRepo& repo, } } -DocumentUpdate::~DocumentUpdate() { } +DocumentUpdate::~DocumentUpdate() = default; bool @@ -92,22 +79,6 @@ DocumentUpdate::operator==(const DocumentUpdate& other) const return true; } -bool -DocumentUpdate::affectsDocumentBody() const -{ - for(const auto & update : _updates) { - if (!update.getField().isHeaderField()) { - return true; - } - } - for (const auto & update : _fieldPathUpdates) { - if (update->affectsDocumentBody(*_type)) { - return true; - } - } - return false; -} - const DocumentType& DocumentUpdate::getType() const { return static_cast<const DocumentType &> (*_type); @@ -125,11 +96,6 @@ DocumentUpdate::addFieldPathUpdate(const FieldPathUpdate::CP& update) { return *this; } -DocumentUpdate* -DocumentUpdate::clone() const { - return new DocumentUpdate(*this); -} - void DocumentUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -183,13 +149,6 @@ DocumentUpdate::applyTo(Document& doc) const } void -DocumentUpdate::serialize42(nbostream &stream) const -{ - VespaDocumentSerializer serializer(stream); - serializer.write42(*this); -} - -void DocumentUpdate::serializeHEAD(nbostream &stream) const { VespaDocumentSerializer serializer(stream); @@ -239,15 +198,13 @@ namespace { DocumentUpdate::UP DocumentUpdate::create42(const DocumentTypeRepo& repo, ByteBuffer& buffer) { - return std::make_unique<DocumentUpdate>(repo, buffer, - SerializeVersion::SERIALIZE_42); + return std::make_unique<DocumentUpdate>(repo, buffer, SerializeVersion::SERIALIZE_42); } DocumentUpdate::UP DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer) { - return std::make_unique<DocumentUpdate>(repo, buffer, - SerializeVersion::SERIALIZE_HEAD); + return std::make_unique<DocumentUpdate>(repo, buffer, SerializeVersion::SERIALIZE_HEAD); } void @@ -333,13 +290,6 @@ DocumentUpdate::deserializeFlags(int sizeAndFlags) } void -DocumentUpdate::onDeserialize42(const DocumentTypeRepo &repo, - ByteBuffer& buffer) -{ - deserialize42(repo, buffer); -} - -void DocumentUpdate::printXml(XmlOutputStream& xos) const { xos << XmlTag("document") diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index e6c39278ee2..a7f9d13d693 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -41,15 +41,9 @@ class Document; * path updates was added, and a new serialization format was * introduced while keeping the old one. */ -class DocumentUpdate : public vespalib::Identifiable, - public Printable, - public XmlSerializable +class DocumentUpdate final : public Printable, public XmlSerializable { -public: - typedef std::unique_ptr<DocumentUpdate> UP; - typedef std::shared_ptr<DocumentUpdate> SP; - typedef std::vector<FieldUpdate> FieldUpdateV; - typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateV; +private: /** * Enum class containing the legal serialization version for * document updates. This version is not encoded in the serialized @@ -59,6 +53,11 @@ public: SERIALIZE_42, // old style format, before vespa 5.0 SERIALIZE_HEAD // new style format, since vespa 5.0 }; +public: + typedef std::unique_ptr<DocumentUpdate> UP; + typedef std::shared_ptr<DocumentUpdate> SP; + typedef std::vector<FieldUpdate> FieldUpdateV; + typedef std::vector<FieldPathUpdate::CP> FieldPathUpdateV; /** * Create old style document update, no support for field path updates. @@ -69,7 +68,16 @@ public: * Create new style document update, possibly with field path updates. */ static DocumentUpdate::UP createHEAD(const DocumentTypeRepo&, ByteBuffer&); - + + /** + * Create a document update from a byte buffer containing a serialized + * document update. Public to allow useage in std::make_unique/shared. + * + * @param repo Document type repo used to find proper document type + * @param buffer The buffer containing the serialized document update + * @param serializeVersion Selector between serialization formats. + */ + DocumentUpdate(const DocumentTypeRepo &repo, ByteBuffer &buffer, SerializeVersion serializeVersion); /** * The document type is not strictly needed, as we know this at applyTo() * time, but search does not use applyTo() code to do the update, and don't @@ -82,18 +90,9 @@ public: */ DocumentUpdate(const DataType &type, const DocumentId& id); - /** - * Create a document update from a byte buffer containing a serialized - * document update. - * - * @param repo Document type repo used to find proper document type - * @param buffer The buffer containing the serialized document update - * @param serializeVersion Selector between serialization formats. - */ - DocumentUpdate(const DocumentTypeRepo &repo, ByteBuffer &buffer, - SerializeVersion serializeVersion); - - ~DocumentUpdate(); + DocumentUpdate(const DocumentUpdate &) = delete; + DocumentUpdate & operator = (const DocumentUpdate &) = delete; + ~DocumentUpdate() override; bool operator==(const DocumentUpdate&) const; bool operator!=(const DocumentUpdate & rhs) const { return ! (*this == rhs); } @@ -107,11 +106,6 @@ public: * type as this. */ void applyTo(Document& doc) const; - - void clear() { _updates.clear(); } - size_t size() const { return _updates.size(); } - const FieldUpdate& operator[](int index) const { return _updates[index]; } - FieldUpdate& operator[](int index) { return _updates[index]; } /** * Add a field update to this document update. @@ -131,26 +125,15 @@ public: /** @return The list of fieldpath updates. */ const FieldPathUpdateV & getFieldPathUpdates() const { return _fieldPathUpdates; } - bool affectsDocumentBody() const; - /** @return The type of document this update is for. */ const DocumentType& getType() const; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void deserialize42(const DocumentTypeRepo&, ByteBuffer&); - void deserializeHEAD(const DocumentTypeRepo&, ByteBuffer&); - - // Deserializable implementation. Kept as search relies on it currently. - virtual void onDeserialize42(const DocumentTypeRepo &repo, ByteBuffer&); - - void serialize42(vespalib::nbostream &stream) const; void serializeHEAD(vespalib::nbostream &stream) const; void printXml(XmlOutputStream&) const override; - virtual DocumentUpdate* clone() const; - /** * Sets whether this update should create the document it updates if that document does not exist. * In this case an empty document is created before the update is applied. @@ -169,7 +152,6 @@ public: int serializeFlags(int size_) const; int16_t getVersion() const { return _version; } - DECLARE_IDENTIFIABLE(DocumentUpdate); private: DocumentId _documentId; // The ID of the document to update. const DataType *_type; // The type of document this update is for. @@ -178,15 +160,9 @@ private: int16_t _version; // Serialization version bool _createIfNonExistent; - /** - * This function exist because search relies on deserialization through - * creating object through empty constructor and calling deserialize. - * - * It is hidden to prevent accidental other usage. - */ - DocumentUpdate(); - int deserializeFlags(int sizeAndFlags); + void deserialize42(const DocumentTypeRepo&, ByteBuffer&); + void deserializeHEAD(const DocumentTypeRepo&, ByteBuffer&); }; } // document diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index 7265329d4d5..239ffff2fef 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -45,7 +45,7 @@ FieldPathUpdate::FieldPathUpdate(stringref fieldPath, stringref whereClause) : _originalWhereClause(whereClause) { } -FieldPathUpdate::~FieldPathUpdate() { } +FieldPathUpdate::~FieldPathUpdate() = default; bool FieldPathUpdate::operator==(const FieldPathUpdate& other) const @@ -76,16 +76,6 @@ FieldPathUpdate::applyTo(Document& doc) const } } -bool -FieldPathUpdate::affectsDocumentBody(const DataType & type) const -{ - FieldPath path; - type.buildFieldPath(path, _originalFieldPath); - if (path.empty() || !path[0].hasField()) return false; - const Field& field = path[0].getFieldRef(); - return !field.isHeaderField(); -} - void FieldPathUpdate::print(std::ostream& out, bool, const std::string& indent) const { diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index c120c8d3979..6629aa74aee 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -64,9 +64,6 @@ public: */ void checkCompatibility(const FieldValue& fv, const DataType & type) const; - /** @return Whether or not the first field path element is a body field */ - bool affectsDocumentBody(const DataType & type) const; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_IDENTIFIABLE_ABSTRACT(FieldPathUpdate); diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp index 57396da15ac..277d6467ae8 100644 --- a/document/src/vespa/document/update/fieldupdate.cpp +++ b/document/src/vespa/document/update/fieldupdate.cpp @@ -16,21 +16,32 @@ FieldUpdate::FieldUpdate(const Field& field) { } -// Construct a field update by deserialization. -FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo, - const DocumentType& type, - ByteBuffer& buffer, - int16_t version) +namespace { + +int readInt(ByteBuffer & buffer) { + int tmp; + buffer.getIntNetwork(tmp); + return tmp; +} + +} + +FieldUpdate::FieldUpdate(const DocumentTypeRepo& repo, const DocumentType& type, ByteBuffer& buffer, int16_t version) : Printable(), - _field(), + _field(type.getField(readInt(buffer))), _updates() { - deserialize(repo, type, buffer, version); + int numUpdates = readInt(buffer); + _updates.reserve(numUpdates); + const DataType& dataType = _field.getDataType(); + for(int i(0); i < numUpdates; i++) { + _updates.emplace_back(ValueUpdate::createInstance(repo, dataType, buffer, version).release()); + } } FieldUpdate::FieldUpdate(const FieldUpdate &) = default; FieldUpdate & FieldUpdate::operator = (const FieldUpdate &) = default; -FieldUpdate::~FieldUpdate() {} +FieldUpdate::~FieldUpdate() = default; bool FieldUpdate::operator==(const FieldUpdate& other) const @@ -77,8 +88,7 @@ FieldUpdate::applyTo(Document& doc) const // Print this field update as a human readable string. void -FieldUpdate::print(std::ostream& out, bool verbose, - const std::string& indent) const +FieldUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "FieldUpdate(" << _field.toString(verbose); for(const auto & update : _updates) { @@ -93,8 +103,8 @@ FieldUpdate::print(std::ostream& out, bool verbose, // Deserialize this field update from the given buffer. void -FieldUpdate::deserialize(const DocumentTypeRepo& repo, - const DocumentType& docType, ByteBuffer& buffer, int16_t version) +FieldUpdate::deserialize(const DocumentTypeRepo& repo, const DocumentType& docType, + ByteBuffer& buffer, int16_t version) { int fieldId; buffer.getIntNetwork(fieldId); diff --git a/document/src/vespa/document/update/fieldupdate.h b/document/src/vespa/document/update/fieldupdate.h index 3f7b3dde3f7..569fb21fe1c 100644 --- a/document/src/vespa/document/update/fieldupdate.h +++ b/document/src/vespa/document/update/fieldupdate.h @@ -49,7 +49,7 @@ public: * @param serializationVersion The serialization version the update was serialized with. */ FieldUpdate(const DocumentTypeRepo& repo, const DocumentType& type, - ByteBuffer& buffer, int16_t serializationVersion); + ByteBuffer& buffer, int16_t version); bool operator==(const FieldUpdate&) const; bool operator!=(const FieldUpdate & rhs) const { return ! (*this == rhs); } diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index 0346cadbf9b..48728ff6057 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -7,6 +7,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/update/fieldpathupdates.h> #include <vespa/documentapi/documentapi.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> using document::DataType; using document::DocumentTypeRepo; @@ -240,6 +241,8 @@ Messages50Test::testRemoveLocationMessage() if (EXPECT_TRUE(obj.get() != NULL)) { RemoveLocationMessage &ref = static_cast<RemoveLocationMessage&>(*obj); EXPECT_EQUAL(string("id.group == \"mygroup\""), ref.getDocumentSelection()); + // FIXME add to wire format, currently hardcoded. + EXPECT_EQUAL(string(document::FixedBucketSpaces::default_space_name()), ref.getBucketSpace()); } } } diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp index b5a4a8306b1..26d85b57522 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -32,7 +32,7 @@ RoutableFactories50::DocumentMessageFactory::decode(document::ByteBuffer &in, co uint32_t loadClass = decodeInt(in); DocumentMessage::UP msg = doDecode(in); - if (msg.get() != NULL) { + if (msg) { msg->setPriority((Priority::Value)pri); msg->setLoadType(loadTypes[loadClass]); } @@ -54,7 +54,7 @@ RoutableFactories50::DocumentReplyFactory::decode(document::ByteBuffer &in, cons uint8_t pri; in.getByte(pri); DocumentReply::UP reply = doDecode(in); - if (reply.get() != NULL) { + if (reply) { reply->setPriority((Priority::Value)pri); } return mbus::Routable::UP(reply.release()); @@ -72,22 +72,17 @@ RoutableFactories50::BatchDocumentUpdateMessageFactory::doDecode(document::ByteB uint64_t userId = (uint64_t)decodeLong(buf); string group = decodeString(buf); - BatchDocumentUpdateMessage* msg; - if (group.length()) { - msg = new BatchDocumentUpdateMessage(group); - } else { - msg = new BatchDocumentUpdateMessage(userId); - } - DocumentMessage::UP retVal(msg); + auto msg = (group.length()) + ? std::make_unique<BatchDocumentUpdateMessage>(group) + : std::make_unique<BatchDocumentUpdateMessage>(userId); uint32_t len = decodeInt(buf); for (uint32_t i = 0; i < len; i++) { - document::DocumentUpdate::SP upd; - upd.reset(document::DocumentUpdate::createHEAD(_repo, buf).release()); + document::DocumentUpdate::SP upd = document::DocumentUpdate::createHEAD(_repo, buf); msg->addUpdate(upd); } - return retVal; + return msg; } bool @@ -111,14 +106,14 @@ RoutableFactories50::BatchDocumentUpdateMessageFactory::doEncode(const DocumentM DocumentReply::UP RoutableFactories50::BatchDocumentUpdateReplyFactory::doDecode(document::ByteBuffer &buf) const { - BatchDocumentUpdateReply* reply = new BatchDocumentUpdateReply(); + auto reply = std::make_unique<BatchDocumentUpdateReply>(); reply->setHighestModificationTimestamp(decodeLong(buf)); std::vector<bool>& notFound = reply->getDocumentsNotFound(); notFound.resize(decodeInt(buf)); for (std::size_t i = 0; i < notFound.size(); ++i) { notFound[i] = decodeBoolean(buf); } - return DocumentReply::UP(reply); + return reply; } bool @@ -126,10 +121,10 @@ RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentRep { const BatchDocumentUpdateReply& reply = static_cast<const BatchDocumentUpdateReply&>(obj); buf.putLong(reply.getHighestModificationTimestamp()); - const std::vector<bool>& notFound = reply.getDocumentsNotFound(); - buf.putInt(notFound.size()); - for (std::size_t i = 0; i < notFound.size(); ++i) { - buf.putBoolean(notFound[i]); + const std::vector<bool>& notFoundV = reply.getDocumentsNotFound(); + buf.putInt(notFoundV.size()); + for (bool notFound : notFoundV) { + buf.putBoolean(notFound); } return true; } @@ -137,34 +132,33 @@ RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentRep DocumentMessage::UP RoutableFactories50::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new CreateVisitorMessage()); - CreateVisitorMessage &msg = static_cast<CreateVisitorMessage&>(*ret); + auto msg = std::make_unique<CreateVisitorMessage>(); - msg.setLibraryName(decodeString(buf)); - msg.setInstanceId(decodeString(buf)); - msg.setControlDestination(decodeString(buf)); - msg.setDataDestination(decodeString(buf)); - msg.setDocumentSelection(decodeString(buf)); - msg.setMaximumPendingReplyCount(decodeInt(buf)); + msg->setLibraryName(decodeString(buf)); + msg->setInstanceId(decodeString(buf)); + msg->setControlDestination(decodeString(buf)); + msg->setDataDestination(decodeString(buf)); + msg->setDocumentSelection(decodeString(buf)); + msg->setMaximumPendingReplyCount(decodeInt(buf)); int32_t len = decodeInt(buf); for (int32_t i = 0; i < len; i++) { int64_t val; buf.getLong(val); // NOT using getLongNetwork - msg.getBuckets().push_back(document::BucketId(val)); + msg->getBuckets().push_back(document::BucketId(val)); } - msg.setFromTimestamp(decodeLong(buf)); - msg.setToTimestamp(decodeLong(buf)); - msg.setVisitRemoves(decodeBoolean(buf)); - msg.setVisitHeadersOnly(decodeBoolean(buf)); - msg.setVisitInconsistentBuckets(decodeBoolean(buf)); - msg.getParameters().deserialize(_repo, buf); - msg.setVisitorDispatcherVersion(50); - msg.setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); - msg.setMaxBucketsPerVisitor(decodeInt(buf)); + msg->setFromTimestamp(decodeLong(buf)); + msg->setToTimestamp(decodeLong(buf)); + msg->setVisitRemoves(decodeBoolean(buf)); + msg->setVisitHeadersOnly(decodeBoolean(buf)); + msg->setVisitInconsistentBuckets(decodeBoolean(buf)); + msg->getParameters().deserialize(_repo, buf); + msg->setVisitorDispatcherVersion(50); + msg->setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); + msg->setMaxBucketsPerVisitor(decodeInt(buf)); - return ret; + return msg; } bool @@ -180,11 +174,8 @@ RoutableFactories50::CreateVisitorMessageFactory::doEncode(const DocumentMessage buf.putInt(msg.getMaximumPendingReplyCount()); buf.putInt(msg.getBuckets().size()); - const std::vector<document::BucketId> &buckets = msg.getBuckets(); - for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - uint64_t val = it->getRawId(); + for (const auto & bucketId : msg.getBuckets()) { + uint64_t val = bucketId.getRawId(); buf.putBytes((const char*)&val, 8); } @@ -208,9 +199,8 @@ RoutableFactories50::CreateVisitorMessageFactory::doEncode(const DocumentMessage DocumentReply::UP RoutableFactories50::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new CreateVisitorReply(DocumentProtocol::REPLY_CREATEVISITOR)); - CreateVisitorReply &reply = static_cast<CreateVisitorReply&>(*ret); - reply.setLastBucket(document::BucketId((uint64_t)decodeLong(buf))); + auto reply = std::make_unique<CreateVisitorReply>(DocumentProtocol::REPLY_CREATEVISITOR); + reply->setLastBucket(document::BucketId((uint64_t)decodeLong(buf))); vdslib::VisitorStatistics vs; vs.setBucketsVisited(decodeInt(buf)); vs.setDocumentsVisited(decodeLong(buf)); @@ -219,9 +209,9 @@ RoutableFactories50::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &b vs.setBytesReturned(decodeLong(buf)); vs.setSecondPassDocumentsReturned(decodeLong(buf)); vs.setSecondPassBytesReturned(decodeLong(buf)); - reply.setVisitorStatistics(vs); + reply->setVisitorStatistics(vs); - return ret; + return reply; } bool @@ -242,10 +232,9 @@ RoutableFactories50::CreateVisitorReplyFactory::doEncode(const DocumentReply &ob DocumentMessage::UP RoutableFactories50::DestroyVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new DestroyVisitorMessage()); - DestroyVisitorMessage &msg = static_cast<DestroyVisitorMessage&>(*ret); - msg.setInstanceId(decodeString(buf)); - return ret; + auto msg = std::make_unique<DestroyVisitorMessage>(); + msg->setInstanceId(decodeString(buf)); + return msg; } bool @@ -257,35 +246,30 @@ RoutableFactories50::DestroyVisitorMessageFactory::doEncode(const DocumentMessag } DocumentReply::UP -RoutableFactories50::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DESTROYVISITOR)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DESTROYVISITOR); } bool -RoutableFactories50::DestroyVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::DestroyVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentMessage::UP RoutableFactories50::DocumentListMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new DocumentListMessage()); - DocumentListMessage &msg = static_cast<DocumentListMessage&>(*ret); - - msg.setBucketId(document::BucketId(decodeLong(buf))); + auto msg = std::make_unique<DocumentListMessage>(); + msg->setBucketId(document::BucketId(decodeLong(buf))); int32_t len = decodeInt(buf); for (int32_t i = 0; i < len; i++) { DocumentListMessage::Entry entry(_repo, buf); - msg.getDocuments().push_back(entry); + msg->getDocuments().push_back(entry); } - return ret; + return msg; } bool @@ -295,40 +279,36 @@ RoutableFactories50::DocumentListMessageFactory::doEncode(const DocumentMessage buf.putLong(msg.getBucketId().getRawId()); buf.putInt(msg.getDocuments().size()); - for (uint32_t i = 0; i < msg.getDocuments().size(); i++) { - int len = msg.getDocuments()[i].getSerializedSize(); + for (const auto & document : msg.getDocuments()) { + int len = document.getSerializedSize(); char *tmp = buf.allocate(len); document::ByteBuffer dbuf(tmp, len); - msg.getDocuments()[i].serialize(dbuf); + document.serialize(dbuf); } return true; } DocumentReply::UP -RoutableFactories50::DocumentListReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::DocumentListReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTLIST)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTLIST); } bool -RoutableFactories50::DocumentListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::DocumentListReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentMessage::UP RoutableFactories50::DocumentSummaryMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new DocumentSummaryMessage()); - DocumentSummaryMessage &msg = static_cast<DocumentSummaryMessage&>(*ret); + auto msg = std::make_unique<DocumentSummaryMessage>(); - msg.deserialize(buf); + msg->deserialize(buf); - return ret; + return msg; } bool @@ -345,34 +325,30 @@ RoutableFactories50::DocumentSummaryMessageFactory::doEncode(const DocumentMessa } DocumentReply::UP -RoutableFactories50::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTSUMMARY)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTSUMMARY); } bool -RoutableFactories50::DocumentSummaryReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::DocumentSummaryReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentMessage::UP RoutableFactories50::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new EmptyBucketsMessage()); - EmptyBucketsMessage &msg = static_cast<EmptyBucketsMessage&>(*ret); + auto msg = std::make_unique<EmptyBucketsMessage>(); int32_t len = decodeInt(buf); std::vector<document::BucketId> buckets(len); for (int32_t i = 0; i < len; ++i) { buckets[i] = document::BucketId(decodeLong(buf)); } - msg.getBucketIds().swap(buckets); + msg->getBucketIds().swap(buckets); - return ret; + return msg; } bool @@ -381,35 +357,28 @@ RoutableFactories50::EmptyBucketsMessageFactory::doEncode(const DocumentMessage const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj); buf.putInt(msg.getBucketIds().size()); - const std::vector<document::BucketId> &buckets = msg.getBucketIds(); - for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - buf.putLong(it->getRawId()); + for (const auto & bucketId : msg.getBucketIds()) { + buf.putLong(bucketId.getRawId()); } return true; } DocumentReply::UP -RoutableFactories50::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_EMPTYBUCKETS)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_EMPTYBUCKETS); } bool -RoutableFactories50::EmptyBucketsReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::EmptyBucketsReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } -bool RoutableFactories50::GetBucketListMessageFactory::encodeBucketSpace( - vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& buf) const { - (void) buf; +bool RoutableFactories50::GetBucketListMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace, + vespalib::GrowableByteBuffer& ) const +{ return (bucketSpace == FixedBucketSpaces::default_space_name()); } @@ -437,18 +406,18 @@ RoutableFactories50::GetBucketListMessageFactory::doEncode(const DocumentMessage DocumentReply::UP RoutableFactories50::GetBucketListReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new GetBucketListReply()); - GetBucketListReply &reply = static_cast<GetBucketListReply&>(*ret); + auto reply = std::make_unique<GetBucketListReply>(); int32_t len = decodeInt(buf); + reply->getBuckets().reserve(len); for (int32_t i = 0; i < len; i++) { GetBucketListReply::BucketInfo info; info._bucket = document::BucketId((uint64_t)decodeLong(buf)); info._bucketInformation = decodeString(buf); - reply.getBuckets().push_back(info); + reply->getBuckets().push_back(info); } - return ret; + return reply; } bool @@ -458,11 +427,9 @@ RoutableFactories50::GetBucketListReplyFactory::doEncode(const DocumentReply &ob const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets(); buf.putInt(buckets.size()); - for (std::vector<GetBucketListReply::BucketInfo>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - buf.putLong(it->_bucket.getRawId()); - buf.putString(it->_bucketInformation); + for (const auto & bucketInfo : buckets) { + buf.putLong(bucketInfo._bucket.getRawId()); + buf.putString(bucketInfo._bucketInformation); } return true; @@ -471,12 +438,11 @@ RoutableFactories50::GetBucketListReplyFactory::doEncode(const DocumentReply &ob DocumentMessage::UP RoutableFactories50::GetBucketStateMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new GetBucketStateMessage()); - GetBucketStateMessage &msg = static_cast<GetBucketStateMessage&>(*ret); + auto msg = std::make_unique<GetBucketStateMessage>(); - msg.setBucketId(document::BucketId((uint64_t)decodeLong(buf))); + msg->setBucketId(document::BucketId((uint64_t)decodeLong(buf))); - return ret; + return msg; } bool @@ -490,16 +456,15 @@ RoutableFactories50::GetBucketStateMessageFactory::doEncode(const DocumentMessag DocumentReply::UP RoutableFactories50::GetBucketStateReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new GetBucketStateReply()); - GetBucketStateReply &reply = static_cast<GetBucketStateReply&>(*ret); + auto reply = std::make_unique<GetBucketStateReply>(); int32_t len = decodeInt(buf); + reply->getBucketState().reserve(len); for (int32_t i = 0; i < len; i++) { - DocumentState state(buf); - reply.getBucketState().push_back(state); + reply->getBucketState().emplace_back(buf); } - return ret; + return reply; } bool @@ -508,11 +473,8 @@ RoutableFactories50::GetBucketStateReplyFactory::doEncode(const DocumentReply &o const GetBucketStateReply &reply = static_cast<const GetBucketStateReply&>(obj); buf.putInt(reply.getBucketState().size()); - const std::vector<DocumentState> &state = reply.getBucketState(); - for (std::vector<DocumentState>::const_iterator it = state.begin(); - it != state.end(); ++it) - { - it->serialize(buf); + for (const auto & state : reply.getBucketState()) { + state.serialize(buf); } return true; @@ -521,13 +483,11 @@ RoutableFactories50::GetBucketStateReplyFactory::doEncode(const DocumentReply &o DocumentMessage::UP RoutableFactories50::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new GetDocumentMessage()); - GetDocumentMessage &msg = static_cast<GetDocumentMessage&>(*ret); - - msg.setDocumentId(decodeDocumentId(buf)); - msg.setFlags(decodeInt(buf)); + auto msg = std::make_unique<GetDocumentMessage>(); + msg->setDocumentId(decodeDocumentId(buf)); + msg->setFlags(decodeInt(buf)); - return ret; + return msg; } bool @@ -544,23 +504,22 @@ RoutableFactories50::GetDocumentMessageFactory::doEncode(const DocumentMessage & DocumentReply::UP RoutableFactories50::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new GetDocumentReply()); - GetDocumentReply &reply = static_cast<GetDocumentReply&>(*ret); + auto reply = std::make_unique<GetDocumentReply>(); bool hasDocument = decodeBoolean(buf); document::Document * document = nullptr; if (hasDocument) { auto doc = std::make_shared<document::Document>(_repo, buf); document = doc.get(); - reply.setDocument(std::move(doc)); + reply->setDocument(std::move(doc)); } int64_t lastModified = decodeLong(buf); - reply.setLastModified(lastModified); + reply->setLastModified(lastModified); if (hasDocument) { document->setLastModified(lastModified); } - return ret; + return reply; } bool @@ -582,12 +541,9 @@ RoutableFactories50::GetDocumentReplyFactory::doEncode(const DocumentReply &obj, DocumentMessage::UP RoutableFactories50::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new MapVisitorMessage()); - MapVisitorMessage &msg = static_cast<MapVisitorMessage&>(*ret); - - msg.getData().deserialize(_repo, buf); - - return ret; + auto msg = std::make_unique<MapVisitorMessage>(); + msg->getData().deserialize(_repo, buf); + return msg; } bool @@ -604,17 +560,14 @@ RoutableFactories50::MapVisitorMessageFactory::doEncode(const DocumentMessage &o } DocumentReply::UP -RoutableFactories50::MapVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::MapVisitorReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MAPVISITOR)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_MAPVISITOR); } bool -RoutableFactories50::MapVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::MapVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } @@ -672,11 +625,10 @@ RoutableFactories50::RemoveDocumentMessageFactory::doEncode(const DocumentMessag DocumentReply::UP RoutableFactories50::RemoveDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new RemoveDocumentReply()); - RemoveDocumentReply &reply = static_cast<RemoveDocumentReply&>(*ret); - reply.setWasFound(decodeBoolean(buf)); - reply.setHighestModificationTimestamp(decodeLong(buf)); - return ret; + auto reply = std::make_unique<RemoveDocumentReply>(); + reply->setWasFound(decodeBoolean(buf)); + reply->setHighestModificationTimestamp(decodeLong(buf)); + return reply; } bool @@ -696,7 +648,10 @@ RoutableFactories50::RemoveLocationMessageFactory::doDecode(document::ByteBuffer document::BucketIdFactory factory; document::select::Parser parser(_repo, factory); - return DocumentMessage::UP(new RemoveLocationMessage(factory, parser, selection)); + auto msg = std::make_unique<RemoveLocationMessage>(factory, parser, selection); + // FIXME bucket space not part of wire format, implicitly limiting to only default space for now. + msg->setBucketSpace(document::FixedBucketSpaces::default_space_name()); + return msg; } bool @@ -710,7 +665,7 @@ RoutableFactories50::RemoveLocationMessageFactory::doEncode(const DocumentMessag DocumentReply::UP RoutableFactories50::RemoveLocationReplyFactory::doDecode(document::ByteBuffer &) const { - return DocumentReply::UP(new DocumentReply(DocumentProtocol::REPLY_REMOVELOCATION)); + return std::make_unique<DocumentReply>(DocumentProtocol::REPLY_REMOVELOCATION); } bool @@ -722,12 +677,9 @@ RoutableFactories50::RemoveLocationReplyFactory::doEncode(const DocumentReply &, DocumentMessage::UP RoutableFactories50::SearchResultMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new SearchResultMessage()); - SearchResultMessage &msg = static_cast<SearchResultMessage&>(*ret); - - msg.deserialize(buf); - - return ret; + auto msg = std::make_unique<SearchResultMessage>(); + msg->deserialize(buf); + return msg; } bool @@ -746,13 +698,11 @@ RoutableFactories50::SearchResultMessageFactory::doEncode(const DocumentMessage DocumentMessage::UP RoutableFactories50::QueryResultMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new QueryResultMessage()); - QueryResultMessage &msg = static_cast<QueryResultMessage&>(*ret); - - msg.getSearchResult().deserialize(buf); - msg.getDocumentSummary().deserialize(buf); + auto msg = std::make_unique<QueryResultMessage>(); + msg->getSearchResult().deserialize(buf); + msg->getDocumentSummary().deserialize(buf); - return ret; + return msg; } bool @@ -770,39 +720,32 @@ RoutableFactories50::QueryResultMessageFactory::doEncode(const DocumentMessage & } DocumentReply::UP -RoutableFactories50::SearchResultReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::SearchResultReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_SEARCHRESULT)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); } bool -RoutableFactories50::SearchResultReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::SearchResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentReply::UP -RoutableFactories50::QueryResultReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::QueryResultReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_QUERYRESULT)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_QUERYRESULT); } bool -RoutableFactories50::QueryResultReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::QueryResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } -bool RoutableFactories50::StatBucketMessageFactory::encodeBucketSpace( - vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& buf) const { - (void) buf; +bool RoutableFactories50::StatBucketMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace, + vespalib::GrowableByteBuffer& ) const +{ return (bucketSpace == FixedBucketSpaces::default_space_name()); } @@ -813,14 +756,13 @@ string RoutableFactories50::StatBucketMessageFactory::decodeBucketSpace(document DocumentMessage::UP RoutableFactories50::StatBucketMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new StatBucketMessage()); - StatBucketMessage &msg = static_cast<StatBucketMessage&>(*ret); + auto msg = std::make_unique<StatBucketMessage>(); - msg.setBucketId(document::BucketId(decodeLong(buf))); - msg.setDocumentSelection(decodeString(buf)); - msg.setBucketSpace(decodeBucketSpace(buf)); + msg->setBucketId(document::BucketId(decodeLong(buf))); + msg->setDocumentSelection(decodeString(buf)); + msg->setBucketSpace(decodeBucketSpace(buf)); - return ret; + return msg; } bool @@ -836,12 +778,9 @@ RoutableFactories50::StatBucketMessageFactory::doEncode(const DocumentMessage &o DocumentReply::UP RoutableFactories50::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new StatBucketReply()); - StatBucketReply &reply = static_cast<StatBucketReply&>(*ret); - - reply.setResults(decodeString(buf)); - - return ret; + auto reply = std::make_unique<StatBucketReply>(); + reply->setResults(decodeString(buf)); + return reply; } bool @@ -853,41 +792,32 @@ RoutableFactories50::StatBucketReplyFactory::doEncode(const DocumentReply &obj, } DocumentMessage::UP -RoutableFactories50::StatDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::StatDocumentMessageFactory::doDecode(document::ByteBuffer &) const { - (void)buf; return DocumentMessage::UP(); // TODO: remove message type } bool -RoutableFactories50::StatDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::StatDocumentMessageFactory::doEncode(const DocumentMessage &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return false; } DocumentReply::UP -RoutableFactories50::StatDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::StatDocumentReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; return DocumentReply::UP(); // TODO: remove reply type } bool -RoutableFactories50::StatDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::StatDocumentReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return false; } void RoutableFactories50::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocumentUpdate(make_shared<document::DocumentUpdate> - (_repo, buf, - document::DocumentUpdate::SerializeVersion:: - SERIALIZE_HEAD)); + msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, buf)); msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf))); msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf))); } @@ -909,11 +839,10 @@ RoutableFactories50::UpdateDocumentMessageFactory::doEncode(const DocumentMessag DocumentReply::UP RoutableFactories50::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new UpdateDocumentReply()); - UpdateDocumentReply &reply = static_cast<UpdateDocumentReply&>(*ret); - reply.setWasFound(decodeBoolean(buf)); - reply.setHighestModificationTimestamp(decodeLong(buf)); - return ret; + auto reply = std::make_unique<UpdateDocumentReply>(); + reply->setWasFound(decodeBoolean(buf)); + reply->setHighestModificationTimestamp(decodeLong(buf)); + return reply; } bool @@ -928,18 +857,18 @@ RoutableFactories50::UpdateDocumentReplyFactory::doEncode(const DocumentReply &o DocumentMessage::UP RoutableFactories50::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &buf) const { - DocumentMessage::UP ret(new VisitorInfoMessage()); - VisitorInfoMessage &msg = static_cast<VisitorInfoMessage&>(*ret); + auto msg = std::make_unique<VisitorInfoMessage>(); int32_t len = decodeInt(buf); + msg->getFinishedBuckets().reserve(len); for (int32_t i = 0; i < len; i++) { int64_t val; buf.getLong(val); // NOT using getLongNetwork - msg.getFinishedBuckets().push_back(document::BucketId(val)); + msg->getFinishedBuckets().emplace_back(val); } - msg.setErrorMessage(decodeString(buf)); + msg->setErrorMessage(decodeString(buf)); - return ret; + return msg; } bool @@ -948,11 +877,8 @@ RoutableFactories50::VisitorInfoMessageFactory::doEncode(const DocumentMessage & const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj); buf.putInt(msg.getFinishedBuckets().size()); - const std::vector<document::BucketId> &buckets = msg.getFinishedBuckets(); - for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - uint64_t val = it->getRawId(); + for (const auto & bucketId : msg.getFinishedBuckets()) { + uint64_t val = bucketId.getRawId(); buf.putBytes((const char*)&val, 8); } buf.putString(msg.getErrorMessage()); @@ -961,29 +887,23 @@ RoutableFactories50::VisitorInfoMessageFactory::doEncode(const DocumentMessage & } DocumentReply::UP -RoutableFactories50::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &buf) const +RoutableFactories50::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &) const { - (void)buf; - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_VISITORINFO)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_VISITORINFO); } bool -RoutableFactories50::VisitorInfoReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +RoutableFactories50::VisitorInfoReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const { - (void)obj; - (void)buf; return true; } DocumentReply::UP RoutableFactories50::WrongDistributionReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new WrongDistributionReply()); - WrongDistributionReply &reply = static_cast<WrongDistributionReply&>(*ret); - - reply.setSystemState(decodeString(buf)); - - return ret; + auto reply = std::make_unique<WrongDistributionReply>(); + reply->setSystemState(decodeString(buf)); + return reply; } bool @@ -1013,19 +933,19 @@ RoutableFactories50::FeedMessageFactory::myEncode(const FeedMessage &msg, vespal DocumentReply::UP RoutableFactories50::FeedReplyFactory::doDecode(document::ByteBuffer &buf) const { - DocumentReply::UP ret(new FeedReply(getType())); - FeedReply &reply = static_cast<FeedReply&>(*ret); + auto reply = std::make_unique<FeedReply>(getType()); - std::vector<FeedAnswer> &answers = reply.getFeedAnswers(); + std::vector<FeedAnswer> &answers = reply->getFeedAnswers(); int32_t len = decodeInt(buf); + answers.reserve(len); for (int32_t i = 0; i < len; ++i) { int32_t typeCode = decodeInt(buf); int32_t wantedIncrement = decodeInt(buf); string recipient = decodeString(buf); string moreInfo = decodeString(buf); - answers.push_back(FeedAnswer(typeCode, wantedIncrement, recipient, moreInfo)); + answers.emplace_back(typeCode, wantedIncrement, recipient, moreInfo); } - return ret; + return reply; } bool @@ -1033,14 +953,11 @@ RoutableFactories50::FeedReplyFactory::doEncode(const DocumentReply &obj, vespal { const FeedReply &reply = static_cast<const FeedReply&>(obj); buf.putInt(reply.getFeedAnswers().size()); - const std::vector<FeedAnswer> &answers = reply.getFeedAnswers(); - for (std::vector<FeedAnswer>::const_iterator it = answers.begin(); - it != answers.end(); ++it) - { - buf.putInt(it->getAnswerCode()); - buf.putInt(it->getWantedIncrement()); - buf.putString(it->getRecipient()); - buf.putString(it->getMoreInfo()); + for (const auto & answer : reply.getFeedAnswers()) { + buf.putInt(answer.getAnswerCode()); + buf.putInt(answer.getWantedIncrement()); + buf.putString(answer.getRecipient()); + buf.putString(answer.getMoreInfo()); } return true; } diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp index ce83a2cd638..c7f3401d3e1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp @@ -17,8 +17,7 @@ RoutableRepository::VersionMap::VersionMap() : { } bool -RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version, - IRoutableFactory::SP factory) +RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version, IRoutableFactory::SP factory) { bool ret = _factoryVersions.find(version) != _factoryVersions.end(); _factoryVersions[version] = factory; @@ -41,7 +40,8 @@ RoutableRepository::VersionMap::getFactory(const vespalib::Version &version) con return IRoutableFactory::SP(); } - return std::max_element(candidates.begin(), candidates.end(), [](auto & lhs, auto & rhs) { return lhs.first.compareTo(rhs.first) <= 0; })->second; + return std::max_element(candidates.begin(), candidates.end(), + [](auto & lhs, auto & rhs) { return lhs.first.compareTo(rhs.first) <= 0; })->second; } RoutableRepository::RoutableRepository(const LoadTypeSet& loadTypes) : @@ -50,7 +50,6 @@ RoutableRepository::RoutableRepository(const LoadTypeSet& loadTypes) : _cache(), _loadTypes(loadTypes) { - // empty } mbus::Routable::UP @@ -65,13 +64,13 @@ RoutableRepository::decode(const vespalib::Version &version, mbus::BlobRef data) int type; in.getIntNetwork(type); IRoutableFactory::SP factory = getFactory(version, type); - if (factory.get() == NULL) { + if (!factory) { LOG(error, "No routable factory found for routable type %d (version %s).", type, version.toString().c_str()); return mbus::Routable::UP(); } mbus::Routable::UP ret = factory->decode(in, _loadTypes); - if (ret.get() == NULL) { + if (!ret) { LOG(error, "Routable factory failed to deserialize routable of type %d (version %s).", type, version.toString().c_str()); @@ -89,7 +88,7 @@ RoutableRepository::encode(const vespalib::Version &version, const mbus::Routabl uint32_t type = obj.getType(); IRoutableFactory::SP factory = getFactory(version, type); - if (factory.get() == NULL) { + if (!factory) { LOG(error, "No routable factory found for routable type %d (version %s).", type, version.toString().c_str()); return mbus::Blob(0); @@ -130,7 +129,7 @@ RoutableRepository::getFactory(const vespalib::Version &version, uint32_t type) return IRoutableFactory::SP(); } IRoutableFactory::SP factory = vit->second.getFactory(version); - if (factory.get() == NULL) { + if (!factory) { return IRoutableFactory::SP(); } _cache[cacheKey] = factory; @@ -141,11 +140,9 @@ uint32_t RoutableRepository::getRoutableTypes(const vespalib::Version &version, std::vector<uint32_t> &out) const { vespalib::LockGuard guard(_lock); - for (TypeMap::const_iterator it = _factoryTypes.begin(); - it != _factoryTypes.end(); ++it) - { - if (it->second.getFactory(version).get() != NULL) { - out.push_back(it->first); + for (const auto & type : _factoryTypes) { + if (type.second.getFactory(version)) { + out.push_back(type.first); } } return _factoryTypes.size(); diff --git a/fat-model-dependencies/OWNERS b/fat-model-dependencies/OWNERS new file mode 100644 index 00000000000..d34761f1ba5 --- /dev/null +++ b/fat-model-dependencies/OWNERS @@ -0,0 +1,2 @@ +gjoranv +hmusum diff --git a/fat-model-dependencies/README b/fat-model-dependencies/README new file mode 100644 index 00000000000..ba71b189db9 --- /dev/null +++ b/fat-model-dependencies/README @@ -0,0 +1,4 @@ +This module contains all dependencies that must be embedded in the config-model-fat bundle. +This artifact should be depended on by config-model-fat and all amended versions of the +fat config model. This allows pulling in the same set of dependencies without duplication +in pom.xml. diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml new file mode 100644 index 00000000000..1415ca6e5aa --- /dev/null +++ b/fat-model-dependencies/pom.xml @@ -0,0 +1,223 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>fat-model-dependencies</artifactId> + <packaging>pom</packaging> + <version>6-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-model</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- Large, and installed separately as part of Vespa --> + <groupId>org.tensorflow</groupId> + <artifactId>libtensorflow_jni</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>provided-dependencies</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-application-package</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configgen</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-bundle</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jrt</artifactId> + </exclusion> + <exclusion> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>simplemetrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>metrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-disc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>yolean</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>documentapi</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vdslib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>messagebus</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>document</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>linguistics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>statistics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>messagebus-disc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-messagebus</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>processing</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>chain</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>docproc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-search</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- OPTIMIZATION: very large (44 MB) and only used for query sorting --> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-search-and-docproc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>logd</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchcore</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>storage</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vsm</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>indexinglanguage</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>searchsummary</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.major-version}</artifactId> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_http_service</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 0c5dd72c968..7f2d1f1eff7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -5,16 +5,19 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.collections.Pair; +import com.yahoo.config.provision.NodeType; import com.yahoo.io.IOUtils; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; +import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; @@ -46,6 +49,7 @@ public class StorageMaintainer { private static final ContainerName NODE_ADMIN = new ContainerName("node-admin"); private static final ObjectMapper objectMapper = new ObjectMapper(); + private final GaugeWrapper numberOfCoredumpsOnHost; private final CounterWrapper numberOfNodeAdminMaintenanceFails; private final DockerOperations dockerOperations; private final ProcessExecuter processExecuter; @@ -54,7 +58,6 @@ public class StorageMaintainer { private final Map<ContainerName, MaintenanceThrottler> maintenanceThrottlerByContainerName = new ConcurrentHashMap<>(); - public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter, MetricReceiverWrapper metricReceiver, Environment environment, Clock clock) { this.dockerOperations = dockerOperations; this.processExecuter = processExecuter; @@ -63,44 +66,98 @@ public class StorageMaintainer { Dimensions dimensions = new Dimensions.Builder().add("role", "docker").build(); numberOfNodeAdminMaintenanceFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.maintenance.fails"); + numberOfCoredumpsOnHost = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.coredumps"); } public void writeMetricsConfig(ContainerName containerName, NodeSpec node) { - final Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode( - containerName, Paths.get("/etc/yamas-agent/")); - - Path vespaCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa"); - SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all") - .withTag("parentHostname", environment.getParentHostHostname()); + List<SecretAgentCheckConfig> configs = new ArrayList<>(); + // host-life Path hostLifeCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life"); - SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath) - .withTag("namespace", "Vespa") + SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath); + configs.add(annotatedCheck(node, hostLifeSchedule)); + + // ntp + Path ntpCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ntp"); + SecretAgentCheckConfig ntpSchedule = new SecretAgentCheckConfig("ntp", 60, ntpCheckPath); + configs.add(annotatedCheck(node, ntpSchedule)); + + // coredumps (except for the done coredumps which is handled by the host) + Path coredumpCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps"); + SecretAgentCheckConfig coredumpSchedule = new SecretAgentCheckConfig("system-coredumps-processing", 300, + coredumpCheckPath, "--application", "system-coredumps-processing", "--lastmin", + "129600", "--crit", "1", "--coredir", environment.pathInNodeUnderVespaHome("var/crash/processing").toString()); + configs.add(annotatedCheck(node, coredumpSchedule)); + + if (node.getNodeType() != NodeType.config) { + // vespa-health + Path vespaHealthCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health"); + SecretAgentCheckConfig vespaHealthSchedule = new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all"); + configs.add(annotatedCheck(node, vespaHealthSchedule)); + + // vespa + Path vespaCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa"); + SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all"); + configs.add(annotatedCheck(node, vespaSchedule)); + } + + if (node.getNodeType() == NodeType.config) { + // configserver + Path configServerCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ymonsb2"); + SecretAgentCheckConfig configServerSchedule = new SecretAgentCheckConfig("configserver", 60, + configServerCheckPath, "-zero", "configserver"); + configs.add(annotatedCheck(node, configServerSchedule)); + + //zkbackupage + Path zkbackupCheckPath = environment.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); + SecretAgentCheckConfig zkbackupSchedule = new SecretAgentCheckConfig("zkbackupage", 300, + zkbackupCheckPath, "-f", environment.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(), + "-m", "150", "-a", "config-zkbackupage"); + configs.add(annotatedCheck(node, zkbackupSchedule)); + } + + if (node.getNodeType() == NodeType.proxy) { + //routing-configage + Path routingAgeCheckPath = environment.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); + SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60, + routingAgeCheckPath, "-f", environment.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf").toString(), + "-m", "90", "-a", "routing-configage"); + configs.add(annotatedCheck(node, routingAgeSchedule)); + + //ssl-check + Path sslCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status"); + SecretAgentCheckConfig sslSchedule = new SecretAgentCheckConfig("ssl-status", 300, + sslCheckPath, "-e", "localhost", "-p", "4443", "-t", "30"); + configs.add(annotatedCheck(node, sslSchedule)); + } + + // Write config and restart yamas-agent + Path yamasAgentFolder = environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/etc/yamas-agent/")); + configs.forEach(s -> IOExceptionUtil.uncheck(() -> s.writeTo(yamasAgentFolder))); + final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"}; + dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent); + } + + private SecretAgentCheckConfig annotatedCheck(NodeSpec node, SecretAgentCheckConfig check) { + check.withTag("namespace", "Vespa") .withTag("role", "tenants") .withTag("flavor", node.getFlavor()) .withTag("canonicalFlavor", node.getCanonicalFlavor()) .withTag("state", node.getState().toString()) .withTag("zone", environment.getZone()) .withTag("parentHostname", environment.getParentHostHostname()); - node.getOwner().ifPresent(owner -> hostLifeSchedule + node.getOwner().ifPresent(owner -> check .withTag("tenantName", owner.getTenant()) .withTag("app", owner.getApplication() + "." + owner.getInstance()) .withTag("applicationName", owner.getApplication()) .withTag("instanceName", owner.getInstance()) .withTag("applicationId", owner.getTenant() + "." + owner.getApplication() + "." + owner.getInstance())); - node.getMembership().ifPresent(membership -> hostLifeSchedule + node.getMembership().ifPresent(membership -> check .withTag("clustertype", membership.getClusterType()) .withTag("clusterid", membership.getClusterId())); - node.getVespaVersion().ifPresent(version -> hostLifeSchedule.withTag("vespaVersion", version)); + node.getVespaVersion().ifPresent(version -> check.withTag("vespaVersion", version)); - try { - vespaSchedule.writeTo(yamasAgentFolder); - hostLifeSchedule.writeTo(yamasAgentFolder); - final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"}; - dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent); - } catch (IOException e) { - throw new RuntimeException("Failed to write secret-agent schedules for " + containerName, e); - } + return check; } public void writeFilebeatConfig(ContainerName containerName, NodeSpec node) { @@ -218,6 +275,14 @@ public class StorageMaintainer { * @param force Set to true to bypass throttling */ public void handleCoreDumpsForContainer(ContainerName containerName, NodeSpec node, boolean force) { + // Sample number of coredumps on the host + try { + numberOfCoredumpsOnHost.sample(Files.list(environment.pathInNodeAdminToDoneCoredumps()).count()); + } catch (IOException e) { + // Ignore for now - this is either test or a misconfiguration + } + + // Return early if throttled if (! getMaintenanceThrottlerFor(containerName).shouldHandleCoredumpsNow() && !force) return; MaintainerExecutor maintainerExecutor = new MaintainerExecutor(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index f7e9c3ca1d8..bd75368a0dc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -148,6 +148,7 @@ public class AthenzCredentialsMaintainer { } } + @SuppressWarnings("deprecation") private VespaUniqueInstanceId getVespaUniqueInstanceId(NodeSpec nodeSpec) { NodeSpec.Membership membership = nodeSpec.getMembership().get(); NodeSpec.Owner owner = nodeSpec.getOwner().get(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java index 12a0496ed2e..203f52a7b70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java @@ -38,6 +38,8 @@ public class CuratorDatabase { /** Whether we should return data from the cache or always read fro ZooKeeper */ private final boolean useCache; + private final Object cacheCreationLock = new Object(); + /** * All keys, to allow reentrancy. * This will grow forever with the number of applications seen, but this should be too slow to be a problem. @@ -94,14 +96,42 @@ public class CuratorDatabase { public Optional<byte[]> getData(Path path) { return getCache().getData(path); } + private static class CacheAndGeneration { + public CacheAndGeneration(CuratorDatabaseCache cache, long generation) + { + this.cache = cache; + this.generation = generation; + } + public boolean expired() { + return generation != cache.generation(); + } + public CuratorDatabaseCache validCache() { + if (expired()) { + throw new IllegalStateException("The cache has generation " + cache.generation() + + " while the root genration counter in zookeeper says " + generation + + ". That is totally unacceptable and must be a sever programming error in my close vicinity."); + } + return cache; + } + + private CuratorDatabaseCache cache; + private long generation; + } + private CacheAndGeneration getCacheSnapshot() { + return new CacheAndGeneration(cache.get(), changeGenerationCounter.get()); + } private CuratorDatabaseCache getCache() { - CuratorDatabaseCache cache = this.cache.get(); - long currentCuratorGeneration = changeGenerationCounter.get(); - if (currentCuratorGeneration != cache.generation()) { // current cache is invalid - start new - cache = newCache(currentCuratorGeneration); - this.cache.set(cache); + CacheAndGeneration cacheAndGeneration = getCacheSnapshot(); + while (cacheAndGeneration.expired()) { + synchronized (cacheCreationLock) { // Prevent a race for creating new caches + cacheAndGeneration = getCacheSnapshot(); + if (cacheAndGeneration.expired()) { + cache.set(newCache(changeGenerationCounter.get())); + cacheAndGeneration = getCacheSnapshot(); + } + } } - return cache; + return cacheAndGeneration.validCache(); } /** Caches must only be instantiated using this method */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java index 6d92f9c4541..bea7973541a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2.filter; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; @@ -11,6 +13,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import java.security.cert.X509Certificate; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.yahoo.vespa.athenz.tls.SubjectAlternativeName.Type.DNS_NAME; @@ -31,9 +34,13 @@ class NodeIdentifier { private final Zone zone; private final NodeRepository nodeRepository; + + private final Supplier<List<Node>> nodeCache; + NodeIdentifier(Zone zone, NodeRepository nodeRepository) { this.zone = zone; this.nodeRepository = nodeRepository; + nodeCache = Suppliers.memoizeWithExpiration(nodeRepository::getNodes, 1, TimeUnit.MINUTES); } NodePrincipal resolveNode(List<X509Certificate> certificateChain) throws NodeIdentifierException { @@ -73,7 +80,7 @@ class NodeIdentifier { private String getHostFromCalypsoCertificate(List<SubjectAlternativeName> sans) { String openstackId = getUniqueInstanceId(sans); - return nodeRepository.getNodes().stream() + return nodeCache.get().stream() .filter(node -> node.openStackId().equals(openstackId)) .map(Node::hostname) .findFirst() diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java index c0cead74f5f..11c7832091b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java @@ -29,6 +29,7 @@ import java.security.cert.X509Certificate; import java.time.Instant; import java.util.Optional; +import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.*; import static com.yahoo.vespa.athenz.tls.KeyAlgorithm.RSA; import static com.yahoo.vespa.athenz.tls.SignatureAlgorithm.SHA256_WITH_RSA; import static java.util.Collections.emptySet; @@ -161,7 +162,7 @@ public class NodeIdentifierTest { Pkcs10Csr csr = Pkcs10CsrBuilder .fromKeypair(new X500Principal("CN=" + TENANT_NODE_IDENTITY), KEYPAIR, SHA256_WITH_RSA) .build(); - VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(clusterIndex, clusterId, INSTANCE_ID, application, tenant, region, environment); + VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(clusterIndex, clusterId, INSTANCE_ID, application, tenant, region, environment, NODE); X509Certificate certificate = X509CertificateBuilder .fromCsr(csr, ATHENZ_YAHOO_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_RSA, 1) .addSubjectAlternativeName(vespaUniqueInstanceId.asDottedString() + ".instanceid.athenz.provider-name.vespa.yahoo.cloud") @@ -72,6 +72,7 @@ <module>documentapi</module> <module>document</module> <module>documentgen-test</module> + <module>fat-model-dependencies</module> <module>fileacquirer</module> <module>filedistribution</module> <module>fsa</module> diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index ee5f29255fb..634f69a3820 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attribute_test"); #include <vespa/config-attributes.h> #include <vespa/document/fieldvalue/document.h> @@ -15,6 +13,7 @@ LOG_SETUP("attribute_test"); #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> @@ -44,6 +43,9 @@ LOG_SETUP("attribute_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_test"); + namespace vespa { namespace config { namespace search {}}} using namespace config; @@ -139,6 +141,7 @@ struct Fixture : Fixture(1) { } + ~Fixture(); void allocAttributeWriter() { _aw = std::make_unique<AttributeWriter>(_m); } @@ -155,8 +158,12 @@ struct Fixture _aw->put(serialNum, doc, lid, immediateCommit, emptyCallback); } void update(SerialNum serialNum, const DocumentUpdate &upd, + DocumentIdT lid, bool immediateCommit, IFieldUpdateCallback & onUpdate) { + _aw->update(serialNum, upd, lid, immediateCommit, emptyCallback, onUpdate); + } + void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit) { - _aw->update(serialNum, upd, lid, immediateCommit, emptyCallback); + _aw->update(serialNum, doc, lid, immediateCommit, emptyCallback); } void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit = true) { _aw->remove(serialNum, lid, immediateCommit, emptyCallback); @@ -172,6 +179,7 @@ struct Fixture } }; +Fixture::~Fixture() = default; TEST_F("require that attribute writer handles put", Fixture) { @@ -442,8 +450,9 @@ TEST_F("require that attribute writer handles update", Fixture) upd.addUpdate(FieldUpdate(upd.getType().getField("a2")) .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10))); + DummyFieldUpdateCallback onUpdate; bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + f.update(2, upd, 1, immediateCommit, onUpdate); attribute::IntegerContent ibuf; ibuf.fill(*a1, 1); @@ -453,9 +462,9 @@ TEST_F("require that attribute writer handles update", Fixture) EXPECT_EQUAL(1u, ibuf.size()); EXPECT_EQUAL(30u, ibuf[0]); - f.update(2, upd, 1, immediateCommit); // same sync token as previous + f.update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous try { - f.update(1, upd, 1, immediateCommit); // lower sync token than previous + f.update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -488,7 +497,8 @@ TEST_F("require that attribute writer handles predicate update", Fixture) EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + DummyFieldUpdateCallback onUpdate; + f.update(2, upd, 1, immediateCommit, onUpdate); EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); } @@ -675,7 +685,8 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + DummyFieldUpdateCallback onUpdate; + f.update(2, upd, 1, immediateCommit, onUpdate); EXPECT_EQUAL(2u, a1->getNumDocs()); EXPECT_TRUE(tensorAttribute != nullptr); tensor2 = tensorAttribute->getTensor(1); @@ -773,6 +784,158 @@ TEST_F("require that AttributeWriter::forceCommit() clears search cache in impor EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); } +struct StructFixtureBase : public Fixture +{ + DocumentType _type; + const Field _valueField; + StructDataType _structFieldType; + + StructFixtureBase() + : Fixture(), + _type("test"), + _valueField("value", 2, *DataType::INT, true), + _structFieldType("struct") + { + addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}, createSerialNum); + _type.addField(_valueField); + _structFieldType.addField(_valueField); + } + ~StructFixtureBase(); + + std::unique_ptr<StructFieldValue> + makeStruct() + { + return std::make_unique<StructFieldValue>(_structFieldType); + } + + std::unique_ptr<StructFieldValue> + makeStruct(const int32_t value) + { + auto ret = makeStruct(); + ret->setValue(_valueField, IntFieldValue(value)); + return ret; + } + + std::unique_ptr<Document> + makeDoc() + { + return std::make_unique<Document>(_type, DocumentId("id::test::1")); + } +}; + +StructFixtureBase::~StructFixtureBase() = default; + +struct StructArrayFixture : public StructFixtureBase +{ + using StructFixtureBase::makeDoc; + const ArrayDataType _structArrayFieldType; + const Field _structArrayField; + + StructArrayFixture() + : StructFixtureBase(), + _structArrayFieldType(_structFieldType), + _structArrayField("array", _structArrayFieldType, true) + { + addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + _type.addField(_structArrayField); + } + ~StructArrayFixture(); + + std::unique_ptr<Document> + makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) + { + auto doc = makeDoc(); + doc->setValue(_valueField, IntFieldValue(value)); + ArrayFieldValue s(_structArrayFieldType); + for (const auto &arrayValue : arrayValues) { + s.add(*makeStruct(arrayValue)); + } + doc->setValue(_structArrayField, s); + return doc; + } + void checkAttrs(uint32_t lid, int32_t value, const std::vector<int32_t> &arrayValues) { + auto valueAttr = _m->getAttribute("value")->getSP(); + auto arrayValueAttr = _m->getAttribute("array.value")->getSP(); + EXPECT_EQUAL(value, valueAttr->getInt(lid)); + attribute::IntegerContent ibuf; + ibuf.fill(*arrayValueAttr, lid); + EXPECT_EQUAL(arrayValues.size(), ibuf.size()); + for (size_t i = 0; i < arrayValues.size(); ++i) { + EXPECT_EQUAL(arrayValues[i], ibuf[i]); + } + } +}; + +StructArrayFixture::~StructArrayFixture() = default; + +TEST_F("require that update with doc argument updates compound attributes (array)", StructArrayFixture) +{ + auto doc = f.makeDoc(10, {11, 12}); + f.put(10, *doc, 1); + TEST_DO(f.checkAttrs(1, 10, {11, 12})); + doc = f.makeDoc(20, {21}); + f.update(11, *doc, 1, true); + TEST_DO(f.checkAttrs(1, 10, {21})); +} + +struct StructMapFixture : public StructFixtureBase +{ + using StructFixtureBase::makeDoc; + const MapDataType _structMapFieldType; + const Field _structMapField; + + StructMapFixture() + : StructFixtureBase(), + _structMapFieldType(*DataType::INT, _structFieldType), + _structMapField("map", _structMapFieldType, true) + { + addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + _type.addField(_structMapField); + } + + std::unique_ptr<Document> + makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) + { + auto doc = makeDoc(); + doc->setValue(_valueField, IntFieldValue(value)); + MapFieldValue s(_structMapFieldType); + for (const auto &mapValue : mapValues) { + s.put(IntFieldValue(mapValue.first), *makeStruct(mapValue.second)); + } + doc->setValue(_structMapField, s); + return doc; + } + void checkAttrs(uint32_t lid, int32_t expValue, const std::map<int32_t, int32_t> &expMap) { + auto valueAttr = _m->getAttribute("value")->getSP(); + auto mapKeyAttr = _m->getAttribute("map.key")->getSP(); + auto mapValueAttr = _m->getAttribute("map.value.value")->getSP(); + EXPECT_EQUAL(expValue, valueAttr->getInt(lid)); + attribute::IntegerContent mapKeys; + mapKeys.fill(*mapKeyAttr, lid); + attribute::IntegerContent mapValues; + mapValues.fill(*mapValueAttr, lid); + EXPECT_EQUAL(expMap.size(), mapValues.size()); + EXPECT_EQUAL(expMap.size(), mapKeys.size()); + size_t i = 0; + for (const auto &expMapElem : expMap) { + EXPECT_EQUAL(expMapElem.first, mapKeys[i]); + EXPECT_EQUAL(expMapElem.second, mapValues[i]); + ++i; + } + } +}; + +TEST_F("require that update with doc argument updates compound attributes (map)", StructMapFixture) +{ + auto doc = f.makeDoc(10, {{1, 11}, {2, 12}}); + f.put(10, *doc, 1); + TEST_DO(f.checkAttrs(1, 10, {{1, 11}, {2, 12}})); + doc = f.makeDoc(20, {{42, 21}}); + f.update(11, *doc, 1, true); + TEST_DO(f.checkAttrs(1, 10, {{42, 21}})); +} + TEST_MAIN() { vespalib::rmdir(test_dir, true); diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp index d13e4207a9d..bad27938d4b 100644 --- a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp @@ -86,19 +86,6 @@ makeStringWeightedSet(const std::vector<std::pair<vespalib::string, int32_t>> &a return result; } -template <typename F1, typename F2> -void -checkFieldPathChange(F1 f1, F2 f2, const vespalib::string &path, bool same) -{ - FieldPath fieldPath1 = f1.makeFieldPath(path); - FieldPath fieldPath2 = f2.makeFieldPath(path); - EXPECT_TRUE(!fieldPath1.empty()); - EXPECT_TRUE(!fieldPath2.empty()); - EXPECT_TRUE(DocumentFieldExtractor::isSupported(fieldPath1)); - EXPECT_TRUE(DocumentFieldExtractor::isSupported(fieldPath2)); - EXPECT_EQUAL(same, DocumentFieldExtractor::isCompatible(fieldPath1, fieldPath2)); -} - } struct FixtureBase @@ -354,26 +341,6 @@ TEST_F("require that unknown field gives null value", FixtureBase(false)) TEST_DO(f.assertExtracted("unknown", std::unique_ptr<FieldValue>())); } -TEST("require that type changes are detected") -{ - TEST_DO(checkFieldPathChange(SimpleFixture(false), SimpleFixture(false), "weight", true)); - TEST_DO(checkFieldPathChange(SimpleFixture(false), SimpleFixture(true), "weight", false)); - TEST_DO(checkFieldPathChange(ArrayFixture(false), ArrayFixture(false), "weight", true)); - TEST_DO(checkFieldPathChange(ArrayFixture(false), ArrayFixture(true), "weight", false)); - TEST_DO(checkFieldPathChange(SimpleFixture(false), ArrayFixture(false), "weight", false)); - TEST_DO(checkFieldPathChange(WeightedSetFixture(false), WeightedSetFixture(false), "weight", true)); - TEST_DO(checkFieldPathChange(WeightedSetFixture(false), WeightedSetFixture(true), "weight", false)); - TEST_DO(checkFieldPathChange(SimpleFixture(false), WeightedSetFixture(false), "weight", false)); - TEST_DO(checkFieldPathChange(ArrayFixture(false), WeightedSetFixture(false), "weight", false)); - TEST_DO(checkFieldPathChange(StructArrayFixture(false), StructArrayFixture(false), "s.weight", true)); - TEST_DO(checkFieldPathChange(StructArrayFixture(false), StructArrayFixture(true), "s.weight", false)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, false), "s.value.weight", true)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(true, false), "s.value.weight", false)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, true), "s.value.weight", false)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, false), "s.key", true)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, true), "s.key", false)); -} - TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 00eb59f120a..5d040024e63 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/attribute/i_attribute_writer.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/searchcore/proton/common/feedtoken.h> @@ -316,21 +317,23 @@ struct MyAttributeWriter : public IAttributeWriter std::set<vespalib::string> _attrs; proton::IAttributeManager::SP _mgr; MyTracer &_tracer; + MyAttributeWriter(MyTracer &tracer); ~MyAttributeWriter(); - virtual std::vector<AttributeVector *> + + std::vector<AttributeVector *> getWritableAttributes() const override { return std::vector<AttributeVector *>(); } - virtual AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override { + AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override { if (_attrs.count(attrName) == 0) { return nullptr; } AttrMap::const_iterator itr = _attrMap.find(attrName); return ((itr == _attrMap.end()) ? nullptr : itr->second.get()); } - virtual void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType) override { + void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { _putSerial = serialNum; _putDocId = doc.getId(); _putLid = lid; @@ -339,8 +342,8 @@ struct MyAttributeWriter : public IAttributeWriter ++_commitCount; } } - virtual void remove(SerialNum serialNum, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType) override { + void remove(SerialNum serialNum, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { _removeSerial = serialNum; _removeLid = lid; _tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit); @@ -348,37 +351,45 @@ struct MyAttributeWriter : public IAttributeWriter ++_commitCount; } } - virtual void remove(const LidVector & lidsToRemove, SerialNum serialNum, - bool immediateCommit, OnWriteDoneType) override { + void remove(const LidVector & lidsToRemove, SerialNum serialNum, + bool immediateCommit, OnWriteDoneType) override { for (uint32_t lid : lidsToRemove) { LOG(info, "MyAttributeAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)", serialNum, lid); _removes.push_back(lid); _tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit); } } - virtual void update(SerialNum serialNum, const document::DocumentUpdate &upd, - DocumentIdT lid, bool, OnWriteDoneType) override { + void update(SerialNum serialNum, const document::DocumentUpdate &upd, + DocumentIdT lid, bool, OnWriteDoneType, IFieldUpdateCallback & onUpdate) override { _updateSerial = serialNum; _updateDocId = upd.getId(); _updateLid = lid; + for (const auto & fieldUpdate : upd.getUpdates()) { + search::AttributeVector * attr = getWritableAttribute(fieldUpdate.getField().getName()); + onUpdate.onUpdateField(fieldUpdate.getField().getName(), attr); + } } - virtual void heartBeat(SerialNum) override { ++_heartBeatCount; } - virtual void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override { + void update(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { (void) serialNum; + (void) doc; + (void) lid; + (void) immediateCommit; + } + void heartBeat(SerialNum) override { ++_heartBeatCount; } + void compactLidSpace(uint32_t wantedLidLimit, SerialNum ) override { _wantedLidLimit = wantedLidLimit; } - virtual const proton::IAttributeManager::SP &getAttributeManager() const override { + const proton::IAttributeManager::SP &getAttributeManager() const override { return _mgr; } void forceCommit(SerialNum serialNum, OnWriteDoneType) override { - (void) serialNum; ++_commitCount; + ++_commitCount; _tracer.traceCommit(attributeAdapterTypeName, serialNum); } - virtual void onReplayDone(uint32_t docIdLimit) override - { - (void) docIdLimit; - } + void onReplayDone(uint32_t ) override { } + bool getHasCompoundAttribute() const override { return false; } }; MyAttributeWriter::MyAttributeWriter(MyTracer &tracer) @@ -396,7 +407,7 @@ MyAttributeWriter::MyAttributeWriter(MyTracer &tracer) cfg3.setTensorType(ValueType::from_spec("tensor(x[10])")); _attrMap["a3"] = search::AttributeFactory::createAttribute("test3", cfg3); } -MyAttributeWriter::~MyAttributeWriter() {} +MyAttributeWriter::~MyAttributeWriter() = default; struct MyTransport : public feedtoken::ITransport { @@ -420,7 +431,7 @@ struct MyResultHandler : public IGenericResultHandler { vespalib::Gate _gate; MyResultHandler() : _gate() {} - virtual void handle(const storage::spi::Result &) override { + void handle(const storage::spi::Result &) override { _gate.countDown(); } void await() { _gate.await(); } @@ -446,7 +457,7 @@ SchemaContext::SchemaContext() : _schema->addSummaryField(Schema::SummaryField("s1", DataType::STRING, CollectionType::SINGLE)); _builder.reset(new DocBuilder(*_schema)); } -SchemaContext::~SchemaContext() {} +SchemaContext::~SchemaContext() = default; struct DocumentContext diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index a6d7f12f199..681d3543ee1 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -23,6 +23,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> @@ -282,8 +283,10 @@ TEST_F("require that we can deserialize old update operations", Fixture) BucketId bucket(toBucket(docId.getGlobalId())); auto upd(f.makeUpdate()); { - UpdateOperation op(UpdateOperation::makeOldUpdate(bucket, Timestamp(10), upd)); - op.serialize(stream); + UpdateOperation op(bucket, Timestamp(10), upd); + op.serializeDocumentOperationOnly(stream); + document::VespaDocumentSerializer serializer(stream); + serializer.write42(*op.getUpdate()); } { UpdateOperation op(FeedOperation::UPDATE_42); diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 62c61406f67..a770fff3f5f 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -61,6 +61,36 @@ void inject_match_phase_limiting(Properties &setup, const vespalib::string &attr setup.import(cfg); } +FakeResult make_elem_result(const std::vector<std::pair<uint32_t,std::vector<uint32_t> > > &match_data) { + FakeResult result; + uint32_t pos_should_be_ignored = 0; + for (const auto &doc: match_data) { + result.doc(doc.first); + for (const auto &elem: doc.second) { + result.elem(elem).pos(++pos_should_be_ignored); + } + } + return result; +} + +vespalib::string make_simple_stack_dump(const vespalib::string &field, + const vespalib::string &term) +{ + QueryBuilder<ProtonNodeTypes> builder; + builder.addStringTerm(term, field, 1, search::query::Weight(1)); + return StackDumpCreator::create(*builder.build()); +} + +vespalib::string make_same_element_stack_dump(const vespalib::string &a1_term, + const vespalib::string &f1_term) +{ + QueryBuilder<ProtonNodeTypes> builder; + builder.addSameElement(2, "ignored field name"); + builder.addStringTerm(a1_term, "a1", 1, search::query::Weight(1)); + builder.addStringTerm(f1_term, "f1", 2, search::query::Weight(1)); + return StackDumpCreator::create(*builder.build()); +} + //----------------------------------------------------------------------------- const uint32_t NUM_DOCS = 1000; @@ -238,6 +268,13 @@ struct MyWorld { searchContext.attr().addResult("a1", term, result); } + void add_same_element_results(const vespalib::string &a1_term, const vespalib::string &f1_0_term) { + auto a1_result = make_elem_result({{10, {1}}, {20, {2}}, {21, {2}}}); + auto f1_0_result = make_elem_result({{10, {2}}, {20, {2}}, {21, {2}}}); + searchContext.attr().addResult("a1", a1_term, a1_result); + searchContext.idx(0).getFake().addResult("f1", f1_0_term, f1_0_result); + } + void basicResults() { searchContext.idx(0).getFake().addResult("f1", "foo", FakeResult() @@ -249,26 +286,32 @@ struct MyWorld { .doc(600).doc(700).doc(800).doc(900)); } - void setStackDump(Request &request, const vespalib::string &field, - const vespalib::string &term) { - QueryBuilder<ProtonNodeTypes> builder; - builder.addStringTerm(term, field, 1, search::query::Weight(1)); - vespalib::string stack_dump = - StackDumpCreator::create(*builder.build()); + void setStackDump(Request &request, const vespalib::string &stack_dump) { request.stackDump.assign(stack_dump.data(), stack_dump.data() + stack_dump.size()); } - SearchRequest::SP createSimpleRequest(const vespalib::string &field, - const vespalib::string &term) + SearchRequest::SP createRequest(const vespalib::string &stack_dump) { SearchRequest::SP request(new SearchRequest); request->setTimeout(60 * fastos::TimeStamp::SEC); - setStackDump(*request, field, term); + setStackDump(*request, stack_dump); request->maxhits = 10; return request; } + SearchRequest::SP createSimpleRequest(const vespalib::string &field, + const vespalib::string &term) + { + return createRequest(make_simple_stack_dump(field, term)); + } + + SearchRequest::SP createSameElementRequest(const vespalib::string &a1_term, + const vespalib::string &f1_term) + { + return createRequest(make_same_element_stack_dump(a1_term, f1_term)); + } + Matcher::SP createMatcher() { return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, 0); } @@ -317,7 +360,7 @@ struct MyWorld { const vespalib::string & term) { DocsumRequest::SP request(new DocsumRequest); - setStackDump(*request, field, term); + setStackDump(*request, make_simple_stack_dump(field, term)); // match a subset of basic result + request for a non-hit (not // sorted on docid) @@ -800,4 +843,14 @@ TEST("require that fields are tagged with data type") { EXPECT_EQUAL(predicate_field->get_data_type(), FieldInfo::DataType::BOOLEANTREE); } +TEST("require that same element search works (note that this does not test/use the attribute element iterator wrapper)") { + MyWorld world; + world.basicSetup(); + world.add_same_element_results("foo", "bar"); + SearchRequest::SP request = world.createSameElementRequest("foo", "bar"); + SearchReply::UP reply = world.performSearch(request, 1); + ASSERT_EQUAL(1u, reply->hits.size()); + EXPECT_EQUAL(document::DocumentId("doc::20").getGlobalId(), reply->hits[0].gid); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 9262f9a7b6f..705a27c7fc3 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -68,24 +68,8 @@ createUpd(const DocumentType& docType, const DocumentId &docId) return document::DocumentUpdate::SP(new document::DocumentUpdate(docType, docId)); } - -document::Document::UP -clone(const document::Document::SP &doc) -{ - return document::Document::UP(doc->clone()); -} - - -document::DocumentUpdate::UP -clone(const document::DocumentUpdate::SP &upd) -{ - return document::DocumentUpdate::UP(upd->clone()); -} - - storage::spi::ClusterState -createClusterState(const storage::lib::State& nodeState = - storage::lib::State::UP) +createClusterState(const storage::lib::State& nodeState = storage::lib::State::UP) { using storage::lib::Distribution; using storage::lib::Node; @@ -99,11 +83,7 @@ createClusterState(const storage::lib::State& nodeState = StorDistributionConfigBuilder dc; cstate.setNodeState(Node(NodeType::STORAGE, 0), - NodeState(NodeType::STORAGE, - nodeState, - "dummy desc", - 1.0, - 1)); + NodeState(NodeType::STORAGE, nodeState, "dummy desc", 1.0, 1)); cstate.setClusterState(State::UP); dc.redundancy = 1; dc.readyCopies = 1; @@ -222,8 +202,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { void handleUpdate(FeedToken token, const Bucket& bucket, Timestamp timestamp, const document::DocumentUpdate::SP& upd) override { - token->setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)), - existingTimestamp > 0); + token->setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)), existingTimestamp > 0); handle(token, bucket, timestamp, upd->getId()); } @@ -312,8 +291,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { return frozen.find(bucket.getBucketId().getId()) != frozen.end(); } bool wasFrozen(const Bucket &bucket) { - return was_frozen.find(bucket.getBucketId().getId()) - != was_frozen.end(); + return was_frozen.find(bucket.getBucketId().getId()) != was_frozen.end(); } }; @@ -335,7 +313,7 @@ HandlerSet::HandlerSet() handler1(static_cast<MyHandler &>(*phandler1.get())), handler2(static_cast<MyHandler &>(*phandler2.get())) {} -HandlerSet::~HandlerSet() {} +HandlerSet::~HandlerSet() = default; DocumentType type1(createDocType("type1", 1)); DocumentType type2(createDocType("type2", 2)); @@ -405,8 +383,8 @@ struct SimpleResourceWriteFilter : public IResourceWriteFilter _message() {} - virtual bool acceptWriteOperation() const override { return _acceptWriteOperation; } - virtual State getAcceptState() const override { + bool acceptWriteOperation() const override { return _acceptWriteOperation; } + State getAcceptState() const override { return IResourceWriteFilter::State(acceptWriteOperation(), _message); } }; @@ -475,8 +453,7 @@ TEST_F("require that getPartitionStates() prepares all handlers", SimpleFixture) TEST_F("require that puts are routed to handler", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.engine.put(bucket1, tstamp1, doc1, context); assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket0, tstamp0, docId0, f.hset.handler2); @@ -485,20 +462,16 @@ TEST_F("require that puts are routed to handler", SimpleFixture) assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket1, tstamp1, docId2, f.hset.handler2); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), - f.engine.put(bucket1, tstamp1, doc3, context)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), + f.engine.put(bucket1, tstamp1, doc3, context)); } TEST_F("require that puts with old id scheme are rejected", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "Old id scheme not supported in " - "elastic mode (doc:old:id-scheme)"), - f.engine.put(bucket1, tstamp1, old_doc, context)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "Old id scheme not supported in elastic mode (doc:old:id-scheme)"), + f.engine.put(bucket1, tstamp1, old_doc, context)); } @@ -508,8 +481,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL( Result(Result::RESOURCE_EXHAUSTED, "Put operation rejected for document 'doc:old:id-scheme': 'Disk is full'"), @@ -520,8 +492,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur TEST_F("require that updates are routed to handler", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1.setExistingTimestamp(tstamp2); UpdateResult ur = f.engine.update(bucket1, tstamp1, upd1, context); assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); @@ -534,9 +505,8 @@ TEST_F("require that updates are routed to handler", SimpleFixture) assertHandler(bucket1, tstamp1, docId2, f.hset.handler2); EXPECT_EQUAL(tstamp3, ur.getExistingTimestamp()); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), - f.engine.update(bucket1, tstamp1, upd3, context)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), + f.engine.update(bucket1, tstamp1, upd3, context)); } @@ -546,8 +516,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL( Result(Result::RESOURCE_EXHAUSTED, @@ -559,8 +528,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix TEST_F("require that removes are routed to handlers", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); RemoveResult rr = f.engine.remove(bucket1, tstamp1, docId3, context); assertHandler(bucket0, tstamp0, docId0, f.hset.handler1); assertHandler(bucket0, tstamp0, docId0, f.hset.handler2); @@ -598,8 +566,7 @@ TEST_F("require that remove is NOT rejected if resource limit is reached", Simpl f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL(RemoveResult(false), f.engine.remove(bucket1, tstamp1, docId1, context)); } @@ -628,8 +595,7 @@ TEST_F("require that setActiveState() is routed to handlers and merged", SimpleF f.hset.handler1.bucketStateResult = Result(Result::TRANSIENT_ERROR, "err1"); f.hset.handler2.bucketStateResult = Result(Result::PERMANENT_ERROR, "err2"); - Result result = f.engine.setActiveState(bucket1, - storage::spi::BucketInfo::NOT_ACTIVE); + Result result = f.engine.setActiveState(bucket1, storage::spi::BucketInfo::NOT_ACTIVE); EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode()); EXPECT_EQUAL("err1, err2", result.getErrorMessage()); EXPECT_EQUAL(storage::spi::BucketInfo::NOT_ACTIVE, f.hset.handler1.lastBucketState); @@ -651,16 +617,12 @@ TEST_F("require that getBucketInfo() is routed to handlers and merged", SimpleFi } -TEST_F("require that createBucket() is routed to handlers and merged", - SimpleFixture) +TEST_F("require that createBucket() is routed to handlers and merged", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - f.hset.handler1._createBucketResult = - Result(Result::TRANSIENT_ERROR, "err1a"); - f.hset.handler2._createBucketResult = - Result(Result::PERMANENT_ERROR, "err2a"); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + f.hset.handler1._createBucketResult = Result(Result::TRANSIENT_ERROR, "err1a"); + f.hset.handler2._createBucketResult = Result(Result::PERMANENT_ERROR, "err2a"); Result result = f.engine.createBucket(bucket1, context); EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode()); @@ -671,8 +633,7 @@ TEST_F("require that createBucket() is routed to handlers and merged", TEST_F("require that deleteBucket() is routed to handlers and merged", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1.deleteBucketResult = Result(Result::TRANSIENT_ERROR, "err1"); f.hset.handler2.deleteBucketResult = Result(Result::PERMANENT_ERROR, "err2"); @@ -691,10 +652,8 @@ TEST_F("require that getModifiedBuckets() is routed to handlers and merged", Sim TEST_F("require that get is sent to all handlers", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId); EXPECT_EQUAL(docId1, f.hset.handler2.lastDocId); @@ -704,8 +663,7 @@ TEST_F("require that get freezes the bucket", SimpleFixture) { EXPECT_FALSE(f.hset.handler1.wasFrozen(bucket1)); EXPECT_FALSE(f.hset.handler2.wasFrozen(bucket1)); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_TRUE(f.hset.handler1.wasFrozen(bucket1)); EXPECT_TRUE(f.hset.handler2.wasFrozen(bucket1)); @@ -717,10 +675,8 @@ TEST_F("require that get returns the first document found", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); f.hset.handler2.setDocument(*doc2, tstamp2); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId); EXPECT_EQUAL(DocumentId(), f.hset.handler2.lastDocId); @@ -732,8 +688,7 @@ TEST_F("require that get returns the first document found", SimpleFixture) { TEST_F("require that createIterator does", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -741,15 +696,13 @@ TEST_F("require that createIterator does", SimpleFixture) { EXPECT_TRUE(result.getIteratorId()); uint64_t max_size = 1024; - IterateResult it_result = - f.engine.iterate(result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(result.getIteratorId(), max_size, context); EXPECT_FALSE(it_result.hasError()); } TEST_F("require that iterator ids are unique", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -764,10 +717,8 @@ TEST_F("require that iterator ids are unique", SimpleFixture) { TEST_F("require that iterate requires valid iterator", SimpleFixture) { uint64_t max_size = 1024; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - IterateResult it_result = f.engine.iterate(IteratorId(1), max_size, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + IterateResult it_result = f.engine.iterate(IteratorId(1), max_size, context); EXPECT_TRUE(it_result.hasError()); EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode()); EXPECT_EQUAL("Unknown iterator with id 1", it_result.getErrorMessage()); @@ -786,16 +737,14 @@ TEST_F("require that iterate returns documents", SimpleFixture) { f.hset.handler2.setDocument(*doc2, tstamp2); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); uint64_t max_size = 1024; CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(result.getIteratorId()); - IterateResult it_result = - f.engine.iterate(result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(result.getIteratorId(), max_size, context); EXPECT_FALSE(it_result.hasError()); EXPECT_EQUAL(2u, it_result.getEntries().size()); } @@ -804,33 +753,28 @@ TEST_F("require that destroyIterator prevents iteration", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(create_result.getIteratorId()); - Result result = f.engine.destroyIterator(create_result.getIteratorId(), - context); + Result result = f.engine.destroyIterator(create_result.getIteratorId(), context); EXPECT_FALSE(result.hasError()); uint64_t max_size = 1024; - IterateResult it_result = - f.engine.iterate(create_result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(create_result.getIteratorId(), max_size, context); EXPECT_TRUE(it_result.hasError()); EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode()); string msg_prefix = "Unknown iterator with id"; - EXPECT_EQUAL(msg_prefix, - it_result.getErrorMessage().substr(0, msg_prefix.size())); + EXPECT_EQUAL(msg_prefix, it_result.getErrorMessage().substr(0, msg_prefix.size())); } TEST_F("require that buckets are frozen during iterator life", SimpleFixture) { EXPECT_FALSE(f.hset.handler1.isFrozen(bucket1)); EXPECT_FALSE(f.hset.handler2.isFrozen(bucket1)); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 0cb260ed9a8..35f5f09fc37 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_writer.h" +#include "ifieldupdatecallback.h" #include "attributemanager.h" #include "document_field_extractor.h" #include <vespa/document/base/exceptions.h> @@ -22,10 +23,34 @@ namespace proton { using LidVector = LidVectorContext::LidVector; +AttributeWriter::WriteField::WriteField(AttributeVector &attribute) + : _fieldPath(), + _attribute(attribute), + _compoundAttribute(false) +{ + const vespalib::string &name = attribute.getName(); + _compoundAttribute = name.find('.') != vespalib::string::npos; +} + +AttributeWriter::WriteField::~WriteField() = default; + +void +AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) +{ + const vespalib::string &name = _attribute.getName(); + FieldPath fp; + try { + docType.buildFieldPath(fp, name); + } catch (document::FieldNotFoundException & e) { + fp = FieldPath(); + } + _fieldPath = std::move(fp); +} + AttributeWriter::WriteContext::WriteContext(uint32_t executorId) : _executorId(executorId), - _fieldPaths(), - _attributes() + _fields(), + _hasCompoundAttribute(false) { } @@ -37,30 +62,20 @@ AttributeWriter::WriteContext::~WriteContext() = default; AttributeWriter::WriteContext &AttributeWriter::WriteContext::operator=(WriteContext &&rhs) = default; void -AttributeWriter::WriteContext::add(AttributeVector *attr) +AttributeWriter::WriteContext::add(AttributeVector &attr) { - _attributes.emplace_back(attr); - _fieldPaths.emplace_back(); + _fields.emplace_back(attr); + if (_fields.back().getCompoundAttribute()) { + _hasCompoundAttribute = true; + } } void AttributeWriter::WriteContext::buildFieldPaths(const DocumentType &docType) { - size_t fieldId = 0; - for (const auto &attrp : _attributes) { - const vespalib::string &name = attrp->getName(); - FieldPath fp; - try { - docType.buildFieldPath(fp, name); - } catch (document::FieldNotFoundException & e) { - fp = FieldPath(); - } - - assert(fieldId < _fieldPaths.size()); - _fieldPaths[fieldId] = std::move(fp); - ++fieldId; + for (auto &field : _fields) { + field.buildFieldPath(docType); } - assert(fieldId == _fieldPaths.size()); } namespace { @@ -200,26 +215,30 @@ class PutTask : public vespalib::Executor::Task const SerialNum _serialNum; const uint32_t _lid; const bool _immediateCommit; + const bool _allAttributes; std::remove_reference_t<AttributeWriter::OnWriteDoneType> _onWriteDone; std::vector<FieldValue::UP> _fieldValues; public: - PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone); + PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone); virtual ~PutTask() override; virtual void run() override; }; -PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone) +PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone) : _wc(wc), _serialNum(serialNum), _lid(lid), _immediateCommit(immediateCommit), + _allAttributes(allAttributes), _onWriteDone(onWriteDone) { - const auto &fieldPaths = _wc.getFieldPaths(); - _fieldValues.reserve(fieldPaths.size()); - for (const auto &fieldPath : fieldPaths) { - FieldValue::UP fv = fieldExtractor.getFieldValue(fieldPath); - _fieldValues.emplace_back(std::move(fv)); + const auto &fields = _wc.getFields(); + _fieldValues.reserve(fields.size()); + for (const auto &field : fields) { + if (_allAttributes || field.getCompoundAttribute()) { + FieldValue::UP fv = fieldExtractor.getFieldValue(field.getFieldPath()); + _fieldValues.emplace_back(std::move(fv)); + } } } @@ -231,13 +250,15 @@ void PutTask::run() { uint32_t fieldId = 0; - const auto &attributes = _wc.getAttributes(); - for (auto attrp : attributes) { - AttributeVector &attr = *attrp; - if (attr.getStatus().getLastSyncToken() < _serialNum) { - applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, _immediateCommit, attr, _onWriteDone); + const auto &fields = _wc.getFields(); + for (auto field : fields) { + if (_allAttributes || field.getCompoundAttribute()) { + AttributeVector &attr = field.getAttribute(); + if (attr.getStatus().getLastSyncToken() < _serialNum) { + applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, _immediateCommit, attr, _onWriteDone); + } + ++fieldId; } - ++fieldId; } } @@ -271,9 +292,9 @@ RemoveTask::~RemoveTask() void RemoveTask::run() { - const auto &attributes = _wc.getAttributes(); - for (auto &attrp : attributes) { - AttributeVector &attr = *attrp; + const auto &fields = _wc.getFields(); + for (auto &field : fields) { + AttributeVector &attr = field.getAttribute(); // Must use <= due to how move operations are handled if (attr.getStatus().getLastSyncToken() <= _serialNum) { applyRemoveToAttribute(_serialNum, _lid, _immediateCommit, attr, _onWriteDone); @@ -303,13 +324,14 @@ public: {} virtual ~BatchRemoveTask() override {} virtual void run() override { - for (auto attr : _writeCtx.getAttributes()) { - if (attr->getStatus().getLastSyncToken() < _serialNum) { + for (auto field : _writeCtx.getFields()) { + auto &attr = field.getAttribute(); + if (attr.getStatus().getLastSyncToken() < _serialNum) { for (auto lidToRemove : _lidsToRemove) { - applyRemoveToAttribute(_serialNum, lidToRemove, false, *attr, _onWriteDone); + applyRemoveToAttribute(_serialNum, lidToRemove, false, attr, _onWriteDone); } if (_immediateCommit) { - attr->commit(_serialNum, _serialNum); + attr.commit(_serialNum, _serialNum); } } } @@ -342,9 +364,9 @@ CommitTask::~CommitTask() void CommitTask::run() { - const auto &attributes = _wc.getAttributes(); - for (auto &attrp : attributes) { - AttributeVector &attr = *attrp; + const auto &fields = _wc.getFields(); + for (auto &field : fields) { + AttributeVector &attr = field.getAttribute(); applyCommit(_serialNum, _onWriteDone, attr); } } @@ -365,7 +387,12 @@ AttributeWriter::setupWriteContexts() (_writeContexts.back().getExecutorId() != fc.getExecutorId())) { _writeContexts.emplace_back(fc.getExecutorId()); } - _writeContexts.back().add(fc.getAttribute()); + _writeContexts.back().add(*fc.getAttribute()); + } + for (const auto &wc : _writeContexts) { + if (wc.getHasCompoundAttribute()) { + _hasCompoundAttribute = true; + } } } @@ -380,12 +407,18 @@ AttributeWriter::buildFieldPaths(const DocumentType & docType, const DataType *d void AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone) + bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone) { + const DataType *dataType(doc.getDataType()); + if (_dataType != dataType) { + buildFieldPaths(doc.getType(), dataType); + } DocumentFieldExtractor extractor(doc); for (const auto &wc : _writeContexts) { - auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, onWriteDone); - _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + if (allAttributes || wc.getHasCompoundAttribute()) { + auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, allAttributes, onWriteDone); + _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + } } } @@ -405,7 +438,8 @@ AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr) _attributeFieldWriter(mgr->getAttributeFieldWriter()), _writableAttributes(mgr->getWritableAttributes()), _writeContexts(), - _dataType(nullptr) + _dataType(nullptr), + _hasCompoundAttribute(false) { setupWriteContexts(); } @@ -432,18 +466,26 @@ void AttributeWriter::put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) { - FieldValue::UP attrVal; LOG(spam, "Handle put: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)", serialNum, doc.getId().toString().c_str(), lid, doc.toString(true).c_str()); - const DataType *dataType(doc.getDataType()); - if (_dataType != dataType) { - buildFieldPaths(doc.getType(), dataType); - } - internalPut(serialNum, doc, lid, immediateCommit, onWriteDone); + internalPut(serialNum, doc, lid, immediateCommit, true, onWriteDone); +} + +void +AttributeWriter::update(SerialNum serialNum, const Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone) +{ + LOG(spam, + "Handle update: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)", + serialNum, + doc.getId().toString().c_str(), + lid, + doc.toString(true).c_str()); + internalPut(serialNum, doc, lid, immediateCommit, false, onWriteDone); } void @@ -465,17 +507,15 @@ AttributeWriter::remove(const LidVector &lidsToRemove, SerialNum serialNum, void AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone) + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) { LOG(debug, "Inspecting update for document %d.", lid); for (const auto &fupd : upd.getUpdates()) { - LOG(debug, "Retrieving guard for attribute vector '%s'.", - fupd.getField().getName().c_str()); - AttributeVector *attrp = - _mgr->getWritableAttribute(fupd.getField().getName()); + LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().c_str()); + AttributeVector *attrp = _mgr->getWritableAttribute(fupd.getField().getName()); + onUpdate.onUpdateField(fupd.getField().getName(), attrp); if (attrp == nullptr) { - LOG(spam, "Failed to find attribute vector %s", - fupd.getField().getName().c_str()); + LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().c_str()); continue; } AttributeVector &attr = *attrp; @@ -484,8 +524,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document if (attr.getStatus().getLastSyncToken() >= serialNum) continue; - LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", - lid, attr.getName().c_str()); + LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attr.getName().c_str()); // NOTE: The lifetime of the field update will be ensured by keeping the document update alive // in a operation done context object. @@ -550,5 +589,11 @@ AttributeWriter::compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) _attributeFieldWriter.sync(); } +bool +AttributeWriter::getHasCompoundAttribute() const +{ + return _hasCompoundAttribute; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index cbda11180c0..dfdafa3bea9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -23,30 +23,44 @@ private: search::ISequencedTaskExecutor &_attributeFieldWriter; const std::vector<search::AttributeVector *> &_writableAttributes; public: + class WriteField + { + FieldPath _fieldPath; + AttributeVector &_attribute; + bool _compoundAttribute; // in array/map of struct + public: + WriteField(AttributeVector &attribute); + ~WriteField(); + AttributeVector &getAttribute() const { return _attribute; } + const FieldPath &getFieldPath() const { return _fieldPath; } + void buildFieldPath(const DocumentType &docType); + bool getCompoundAttribute() const { return _compoundAttribute; } + }; class WriteContext { uint32_t _executorId; - std::vector<FieldPath> _fieldPaths; - std::vector<AttributeVector *> _attributes; + std::vector<WriteField> _fields; + bool _hasCompoundAttribute; public: WriteContext(uint32_t executorId); WriteContext(WriteContext &&rhs); ~WriteContext(); WriteContext &operator=(WriteContext &&rhs); void buildFieldPaths(const DocumentType &docType); - void add(AttributeVector *attr); + void add(AttributeVector &attr); uint32_t getExecutorId() const { return _executorId; } - const std::vector<FieldPath> &getFieldPaths() const { return _fieldPaths; } - const std::vector<AttributeVector *> &getAttributes() const { return _attributes; } + const std::vector<WriteField> &getFields() const { return _fields; } + bool getHasCompoundAttribute() const { return _hasCompoundAttribute; } }; private: std::vector<WriteContext> _writeContexts; const DataType *_dataType; + bool _hasCompoundAttribute; void setupWriteContexts(); void buildFieldPaths(const DocumentType &docType, const DataType *dataType); void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone); + bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone); void internalRemove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone); @@ -68,6 +82,8 @@ public: void remove(const LidVector &lidVector, SerialNum serialNum, bool immediateCommit, OnWriteDoneType onWriteDone) override; void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) override; + void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) override; void heartBeat(SerialNum serialNum) override; void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override; @@ -76,7 +92,8 @@ public: } void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; - virtual void onReplayDone(uint32_t docIdLimit) override; + void onReplayDone(uint32_t docIdLimit) override; + bool getHasCompoundAttribute() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp index 143441eaae9..46f3fdeff67 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp @@ -146,23 +146,6 @@ DocumentFieldExtractor::isSupported(const FieldPath &fieldPath) return true; } -bool -DocumentFieldExtractor::isCompatible(const FieldPath &fieldPath1, const FieldPath &fieldPath2) -{ - if (fieldPath1.size() != fieldPath2.size()) { - return false; - } - uint32_t arrayIndex = 0; - for (const auto &fieldPathEntry1 : fieldPath1) { - const auto &fieldPathEntry2 = fieldPath2[arrayIndex++]; - if (fieldPathEntry1->getType() != fieldPathEntry2.getType() || - fieldPathEntry1->getDataType() != fieldPathEntry2.getDataType()) { - return false; - } - } - return true; -} - const FieldValue * DocumentFieldExtractor::getCachedFieldValue(const FieldPathEntry &fieldPathEntry) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h index 71f020a582c..48e2da9c4c6 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h @@ -41,15 +41,6 @@ public: * Check if fieldPath is in a supported form. */ static bool isSupported(const document::FieldPath &fieldPath); - - /** - * Check if two field paths are compatible, i.e. same types in whole path - * and same data type would be returned from getFieldValue(). This is - * meant to be used when document type in received document doesn't match - * the document type for the current config (can happen right before and - * after live config change when validation override is used). - */ - static bool isCompatible(const document::FieldPath &fieldPath1, const document::FieldPath &fieldPath2); }; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h index abcf132d537..7a557b17964 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h @@ -1,18 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> +#include "i_attribute_manager.h" +#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/query/base.h> #include <vespa/searchlib/common/serialnum.h> -#include <vespa/searchcore/proton/attribute/i_attribute_manager.h> -#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/documentupdate.h> namespace search { class IDestructorCallback; } namespace proton { +class IFieldUpdateCallback; + /** * Interface for an attribute writer that handles writes in form of put, update and remove * to an underlying set of attribute vectors. @@ -31,10 +33,8 @@ public: virtual ~IAttributeWriter() {} - virtual std::vector<search::AttributeVector *> - getWritableAttributes() const = 0; - virtual search::AttributeVector * - getWritableAttribute(const vespalib::string &attrName) const = 0; + virtual std::vector<search::AttributeVector *> getWritableAttributes() const = 0; + virtual search::AttributeVector *getWritableAttribute(const vespalib::string &attrName) const = 0; virtual void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) = 0; virtual void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit, @@ -46,6 +46,11 @@ public: * The OnWriteDoneType instance should ensure the lifetime of the given DocumentUpdate instance. */ virtual void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) = 0; + /* + * Update the underlying compound attributes based on updated document. + */ + virtual void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) = 0; virtual void heartBeat(SerialNum serialNum) = 0; /** @@ -60,6 +65,8 @@ public: virtual void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0; virtual void onReplayDone(uint32_t docIdLimit) = 0; + + virtual bool getHasCompoundAttribute() const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h b/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h new file mode 100644 index 00000000000..ffb8555cd2c --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace search { class AttributeVector; } + +namespace proton { + +struct IFieldUpdateCallback { + virtual ~IFieldUpdateCallback() { } + virtual void onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) = 0; +}; + +struct DummyFieldUpdateCallback : IFieldUpdateCallback { + void onUpdateField(vespalib::stringref, const search::AttributeVector *) override {} +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp index b2567527560..37b23449315 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp @@ -26,9 +26,7 @@ DocumentOperation::DocumentOperation(Type type) } -DocumentOperation::DocumentOperation(Type type, - const BucketId &bucketId, - const Timestamp ×tamp) +DocumentOperation::DocumentOperation(Type type, const BucketId &bucketId, const Timestamp ×tamp) : FeedOperation(type), _bucketId(bucketId), _timestamp(timestamp), @@ -62,7 +60,12 @@ vespalib::string DocumentOperation::docArgsToString() const { } void -DocumentOperation::serialize(vespalib::nbostream &os) const +DocumentOperation::serialize(vespalib::nbostream &os) const { + serializeDocumentOperationOnly(os); +} + +void +DocumentOperation::serializeDocumentOperationOnly(vespalib::nbostream &os) const { os << _bucketId; os << _timestamp; @@ -74,8 +77,7 @@ DocumentOperation::serialize(vespalib::nbostream &os) const void -DocumentOperation::deserialize(vespalib::nbostream &is, - const DocumentTypeRepo &) +DocumentOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &) { is >> _bucketId; is >> _timestamp; @@ -85,4 +87,8 @@ DocumentOperation::deserialize(vespalib::nbostream &is, is >> _prevTimestamp; } + DbDocumentId DocumentOperation::getDbDocumentId() const { + return _dbdId; + } + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h index bc961c580fc..9a823c553bd 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h @@ -22,124 +22,35 @@ protected: DocumentOperation(Type type); - DocumentOperation(Type type, - const document::BucketId &bucketId, + DocumentOperation(Type type, const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp); void assertValidBucketId(const document::DocumentId &docId) const; vespalib::string docArgsToString() const; public: - virtual - ~DocumentOperation() - { - } - - const - document::BucketId & - getBucketId() const - { - return _bucketId; - } - - storage::spi::Timestamp - getTimestamp() const - { - return _timestamp; - } - - search::DocumentIdT - getLid() const - { - return _dbdId.getLid(); - } - - search::DocumentIdT - getPrevLid() const - { - return _prevDbdId.getLid(); - } - - uint32_t - getSubDbId() const - { - return _dbdId.getSubDbId(); - } - - uint32_t - getPrevSubDbId() const - { - return _prevDbdId.getSubDbId(); - } - - bool - getValidDbdId() const - { - return _dbdId.valid(); - } - - bool - getValidDbdId(uint32_t subDbId) const - { - return _dbdId.valid() && _dbdId.getSubDbId() == subDbId; - } - - bool - getValidPrevDbdId() const - { - return _prevDbdId.valid(); - } - - bool - getValidPrevDbdId(uint32_t subDbId) const - { - return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId; - } - - bool - changedDbdId() const - { - return _dbdId != _prevDbdId; - } - bool - getPrevMarkedAsRemoved() const - { - return _prevMarkedAsRemoved; - } - - void - setPrevMarkedAsRemoved(bool prevMarkedAsRemoved) - { - _prevMarkedAsRemoved = prevMarkedAsRemoved; - } - - DbDocumentId - getDbDocumentId() const - { - return _dbdId; - } - - DbDocumentId - getPrevDbDocumentId() const - { - return _prevDbdId; - } - - void - setDbDocumentId(DbDocumentId dbdId) - { - _dbdId = dbdId; - } - - void - setPrevDbDocumentId(DbDocumentId prevDbdId) - { - _prevDbdId = prevDbdId; - } - - search::DocumentIdT - getNewOrPrevLid(uint32_t subDbId) const - { + ~DocumentOperation() override {} + const document::BucketId &getBucketId() const { return _bucketId; } + storage::spi::Timestamp getTimestamp() const { return _timestamp; } + + search::DocumentIdT getLid() const { return _dbdId.getLid(); } + search::DocumentIdT getPrevLid() const { return _prevDbdId.getLid(); } + uint32_t getSubDbId() const { return _dbdId.getSubDbId(); } + uint32_t getPrevSubDbId() const { return _prevDbdId.getSubDbId(); } + bool getValidDbdId() const { return _dbdId.valid(); } + bool getValidDbdId(uint32_t subDbId) const { return _dbdId.valid() && _dbdId.getSubDbId() == subDbId; } + bool getValidPrevDbdId() const { return _prevDbdId.valid(); } + bool getValidPrevDbdId(uint32_t subDbId) const { return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId; } + bool changedDbdId() const { return _dbdId != _prevDbdId; } + bool getPrevMarkedAsRemoved() const { return _prevMarkedAsRemoved; } + void setPrevMarkedAsRemoved(bool prevMarkedAsRemoved) { _prevMarkedAsRemoved = prevMarkedAsRemoved; } + DbDocumentId getDbDocumentId() const; + DbDocumentId getPrevDbDocumentId() const { return _prevDbdId; } + + void setDbDocumentId(DbDocumentId dbdId) { _dbdId = dbdId; } + void setPrevDbDocumentId(DbDocumentId prevDbdId) { _prevDbdId = prevDbdId; } + + search::DocumentIdT getNewOrPrevLid(uint32_t subDbId) const { if (getValidDbdId() && getSubDbId() == subDbId) return getLid(); if (getValidPrevDbdId() && getPrevSubDbId() == subDbId) @@ -147,51 +58,34 @@ public: return 0; } - bool - getValidNewOrPrevDbdId() const - { + bool getValidNewOrPrevDbdId() const { return getValidDbdId() || getValidPrevDbdId(); } - bool - notMovingLidInSameSubDb() const - { + bool notMovingLidInSameSubDb() const { return !getValidDbdId() || !getValidPrevDbdId() || getSubDbId() != getPrevSubDbId() || getLid() == getPrevLid(); } - bool - movingLidIfInSameSubDb() const - { + bool movingLidIfInSameSubDb() const { return !getValidDbdId() || !getValidPrevDbdId() || getSubDbId() != getPrevSubDbId() || getLid() != getPrevLid(); } - storage::spi::Timestamp - getPrevTimestamp() const - { - return _prevTimestamp; - } - - void - setPrevTimestamp(storage::spi::Timestamp prevTimestamp) - { - _prevTimestamp = prevTimestamp; - } - - virtual void - serialize(vespalib::nbostream &os) const override; + storage::spi::Timestamp getPrevTimestamp() const { return _prevTimestamp; } + void setPrevTimestamp(storage::spi::Timestamp prevTimestamp) { _prevTimestamp = prevTimestamp; } - virtual void - deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) override; + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; uint32_t getSerializedDocSize() const { return _serializedDocSize; } + + // Provided as a hook for tests. + void serializeDocumentOperationOnly(vespalib::nbostream &os) const; }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h index e44b7874c12..8a00c739126 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h @@ -52,8 +52,7 @@ public: void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; } SerialNum getSerialNum() const { return _serialNum; } virtual void serialize(vespalib::nbostream &os) const = 0; - virtual void deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) = 0; + virtual void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) = 0; virtual vespalib::string toString() const = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index beaf719dc5c..3468cc68ced 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -28,21 +28,15 @@ UpdateOperation::UpdateOperation(Type type) } -UpdateOperation::UpdateOperation(Type type, - const BucketId &bucketId, - const Timestamp ×tamp, - const DocumentUpdate::SP &upd) - : DocumentOperation(type, - bucketId, - timestamp), +UpdateOperation::UpdateOperation(Type type, const BucketId &bucketId, + const Timestamp ×tamp, const DocumentUpdate::SP &upd) + : DocumentOperation(type, bucketId, timestamp), _upd(upd) { } -UpdateOperation::UpdateOperation(const BucketId &bucketId, - const Timestamp ×tamp, - const DocumentUpdate::SP &upd) +UpdateOperation::UpdateOperation(const BucketId &bucketId, const Timestamp ×tamp, const DocumentUpdate::SP &upd) : UpdateOperation(FeedOperation::UPDATE, bucketId, timestamp, upd) { } @@ -50,20 +44,15 @@ UpdateOperation::UpdateOperation(const BucketId &bucketId, void UpdateOperation::serializeUpdate(vespalib::nbostream &os) const { - if (getType() == FeedOperation::UPDATE_42) { - _upd->serialize42(os); - } else { - _upd->serializeHEAD(os); - } + assert(getType() == UPDATE); + _upd->serializeHEAD(os); } void UpdateOperation::deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) { document::ByteBuffer buf(is.peek(), is.size()); - using Version = DocumentUpdate::SerializeVersion; - Version version = ((getType() == FeedOperation::UPDATE_42) ? Version::SERIALIZE_42 : Version::SERIALIZE_HEAD); - DocumentUpdate::SP update(std::make_shared<DocumentUpdate>(repo, buf, version)); + DocumentUpdate::UP update = (getType() == UPDATE_42) ? DocumentUpdate::create42(repo, buf) : DocumentUpdate::createHEAD(repo, buf); is.adjustReadPos(buf.getPos()); _upd = std::move(update); } @@ -78,8 +67,7 @@ UpdateOperation::serialize(vespalib::nbostream &os) const void -UpdateOperation::deserialize(vespalib::nbostream &is, - const DocumentTypeRepo &repo) +UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &repo) { DocumentOperation::deserialize(is, repo); try { @@ -108,12 +96,4 @@ vespalib::string UpdateOperation::toString() const { docArgsToString().c_str()); } -UpdateOperation -UpdateOperation::makeOldUpdate(const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, - const document::DocumentUpdate::SP &upd) -{ - return UpdateOperation(FeedOperation::UPDATE_42, bucketId, timestamp, upd); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h index 7886231af82..99dcbfbce6c 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h @@ -15,8 +15,7 @@ class UpdateOperation : public DocumentOperation private: using DocumentUpdateSP = std::shared_ptr<document::DocumentUpdate>; DocumentUpdateSP _upd; - UpdateOperation(Type type, - const document::BucketId &bucketId, + UpdateOperation(Type type, const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); void serializeUpdate(vespalib::nbostream &os) const; @@ -27,15 +26,12 @@ public: UpdateOperation(const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); - virtual ~UpdateOperation() {} + ~UpdateOperation() override {} const DocumentUpdateSP &getUpdate() const { return _upd; } void serialize(vespalib::nbostream &os) const override; void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; void deserializeUpdate(const document::DocumentTypeRepo &repo); - virtual vespalib::string toString() const override; - static UpdateOperation makeOldUpdate(const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, - const DocumentUpdateSP &upd); + vespalib::string toString() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index 78084f29742..0dee7adfa49 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -27,6 +27,7 @@ vespa_add_library(searchcore_matching STATIC ranking_constants.cpp requestcontext.cpp result_processor.cpp + same_element_builder.cpp search_session.cpp session_manager_explorer.cpp sessionmanager.cpp diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index 165fc67179a..268fe63ba4c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -2,6 +2,8 @@ #include "querynodes.h" #include "blueprintbuilder.h" +#include "termdatafromnode.h" +#include "same_element_builder.h" #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> @@ -98,6 +100,15 @@ private: n.setDocumentFrequency(_result->getState().estimate().estHits, _context.getDocIdLimit()); } + void buildSameElement(ProtonSameElement &n) { + SameElementBuilder builder(_requestContext, _context); + for (size_t i = 0; i < n.getChildren().size(); ++i) { + search::query::Node &node = *n.getChildren()[i]; + builder.add_child(node); + } + _result = builder.build(); + } + template <typename NodeType> void buildTerm(NodeType &n) { FieldSpecList indexFields; @@ -131,7 +142,7 @@ protected: void visit(ProtonRank &n) override { buildIntermediate(new RankBlueprint(), n); } void visit(ProtonNear &n) override { buildIntermediate(new NearBlueprint(n.getDistance()), n); } void visit(ProtonONear &n) override { buildIntermediate(new ONearBlueprint(n.getDistance()), n); } - void visit(ProtonSameElement &n) override { buildIntermediate(nullptr /*new SameElementBlueprint())*/, n); } + void visit(ProtonSameElement &n) override { buildSameElement(n); } void visit(ProtonWeightedSetTerm &n) override { buildTerm(n); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h index eb45a735780..44cd6ffabfd 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h @@ -20,4 +20,3 @@ struct BlueprintBuilder { }; } - diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp new file mode 100644 index 00000000000..d3a0ec4726f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp @@ -0,0 +1,96 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "same_element_builder.h" +#include "querynodes.h" +#include <vespa/searchlib/query/tree/customtypevisitor.h> +#include <vespa/searchlib/queryeval/leaf_blueprints.h> + +using search::queryeval::Blueprint; +using search::queryeval::EmptyBlueprint; +using search::queryeval::FieldSpecList; +using search::queryeval::IRequestContext; +using search::queryeval::SameElementBlueprint; +using search::queryeval::Searchable; + +namespace proton::matching { + +namespace { + +class SameElementBuilderVisitor : public search::query::CustomTypeVisitor<ProtonNodeTypes> +{ +private: + const IRequestContext &_requestContext; + ISearchContext &_context; + SameElementBlueprint &_result; + +public: + SameElementBuilderVisitor(const IRequestContext &requestContext, ISearchContext &context, SameElementBlueprint &result) + : _requestContext(requestContext), + _context(context), + _result(result) {} + + template <class TermNode> + void visitTerm(const TermNode &n) { + if (n.numFields() == 1) { + const ProtonTermData::FieldEntry &field = n.field(0); + assert(field.getFieldId() != search::fef::IllegalFieldId); + assert(field.getHandle() == search::fef::IllegalHandle); + FieldSpecList field_spec; + field_spec.add(_result.getNextChildField(field.field_name, field.getFieldId())); + Searchable &searchable = field.attribute_field ? _context.getAttributes() : _context.getIndexes(); + _result.addTerm(searchable.createBlueprint(_requestContext, field_spec, n)); + } + } + + void visit(ProtonAnd &) override {} + void visit(ProtonAndNot &) override {} + void visit(ProtonNear &) override {} + void visit(ProtonONear &) override {} + void visit(ProtonOr &) override {} + void visit(ProtonRank &) override {} + void visit(ProtonWeakAnd &) override {} + void visit(ProtonSameElement &) override {} + + void visit(ProtonWeightedSetTerm &) override {} + void visit(ProtonDotProduct &) override {} + void visit(ProtonWandTerm &) override {} + void visit(ProtonPhrase &) override {} + void visit(ProtonEquiv &) override {} + + void visit(ProtonNumberTerm &n) override { visitTerm(n); } + void visit(ProtonLocationTerm &n) override { visitTerm(n); } + void visit(ProtonPrefixTerm &n) override { visitTerm(n); } + void visit(ProtonRangeTerm &n) override { visitTerm(n); } + void visit(ProtonStringTerm &n) override { visitTerm(n); } + void visit(ProtonSubstringTerm &n) override { visitTerm(n); } + void visit(ProtonSuffixTerm &n) override { visitTerm(n); } + void visit(ProtonPredicateQuery &) override {} + void visit(ProtonRegExpTerm &n) override { visitTerm(n); } +}; + +} // namespace proton::matching::<unnamed> + +SameElementBuilder::SameElementBuilder(const search::queryeval::IRequestContext &requestContext, ISearchContext &context) + : _requestContext(requestContext), + _context(context), + _result(std::make_unique<SameElementBlueprint>()) +{ +} + +void +SameElementBuilder::add_child(search::query::Node &node) +{ + SameElementBuilderVisitor visitor(_requestContext, _context, *_result); + node.accept(visitor); +} + +Blueprint::UP +SameElementBuilder::build() +{ + if (!_result || _result->terms().empty()) { + return std::make_unique<EmptyBlueprint>(); + } + return std::move(_result); +} + +} // namespace proton::matching diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h new file mode 100644 index 00000000000..945bb9a97f6 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h @@ -0,0 +1,24 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "isearchcontext.h" +#include <vespa/searchlib/query/tree/node.h> +#include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/same_element_blueprint.h> + +namespace proton::matching { + +class SameElementBuilder +{ +private: + const search::queryeval::IRequestContext &_requestContext; + ISearchContext &_context; + std::unique_ptr<search::queryeval::SameElementBlueprint> _result; +public: + SameElementBuilder(const search::queryeval::IRequestContext &requestContext, ISearchContext &context); + void add_child(search::query::Node &node); + search::queryeval::Blueprint::UP build(); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index 2b2849d025c..78733b14aaa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -13,23 +13,6 @@ using search::index::Schema; namespace proton { -FastAccessFeedView::UpdateScope -FastAccessFeedView::getUpdateScope(const DocumentUpdate &upd) -{ - UpdateScope updateScope; - for (const auto &update : upd.getUpdates()) { - const vespalib::string &fieldName = update.getField().getName(); - if (!fastPartialUpdateAttribute(fieldName)) { - updateScope._nonAttributeFields = true; - break; - } - } - if (!upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; - } - return updateScope; -} - /** * NOTE: For both put, update and remove we only need to pass the 'onWriteDone' * instance when we are going to commit as part of handling the operation. @@ -47,9 +30,18 @@ FastAccessFeedView::putAttributes(SerialNum serialNum, search::DocumentIdT lid, void FastAccessFeedView::updateAttributes(SerialNum serialNum, search::DocumentIdT lid, const DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate) +{ + _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone, onUpdate); +} + +void +FastAccessFeedView::updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone) { - _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone); + if (_attributeWriter->getHasCompoundAttribute()) { + _attributeWriter->update(serialNum, *doc.get(), lid, immediateCommit, onWriteDone); + } } void @@ -107,19 +99,4 @@ FastAccessFeedView::sync() _writeService.attributeFieldWriter().sync(); } -bool -FastAccessFeedView::fastPartialUpdateAttribute(const vespalib::string &fieldName) const { - search::AttributeVector *attribute = _attributeWriter->getWritableAttribute(fieldName); - if (attribute == nullptr) { - // Partial update to non-attribute field must update document - return false; - } - search::attribute::BasicType::Type attrType = attribute->getBasicType(); - // Partial update to tensor, predicate or reference attribute - // must update document - return ((attrType != search::attribute::BasicType::Type::PREDICATE) && - (attrType != search::attribute::BasicType::Type::TENSOR) && - (attrType != search::attribute::BasicType::Type::REFERENCE)); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h index e1b0cf83f64..3af97b4ecb9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h @@ -39,14 +39,13 @@ private: const IAttributeWriter::SP _attributeWriter; DocIdLimit &_docIdLimit; - UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override; - void putAttributes(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, bool immediateCommit, OnPutDoneType onWriteDone) override; void updateAttributes(SerialNum serialNum, search::DocumentIdT lid, const document::DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate) override; + void updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone) override; - void removeAttributes(SerialNum serialNum, search::DocumentIdT lid, bool immediateCommit, OnRemoveDoneType onWriteDone) override; @@ -73,8 +72,6 @@ public: void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; void sync() override; - - bool fastPartialUpdateAttribute(const vespalib::string &fieldName) const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 0c87d24899d..4cda07eee8b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -100,7 +100,7 @@ void SearchableFeedView::performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc futureDoc, bool immediateCommit, OnOperationDoneType onWriteDone) { - Document::UP doc = std::move(futureDoc.get()); + const auto &doc = futureDoc.get(); if (doc) { performIndexPut(serialNum, lid, *doc, immediateCommit, onWriteDone); } @@ -118,29 +118,6 @@ SearchableFeedView::performIndexHeartBeat(SerialNum serialNum) _indexWriter->heartBeat(serialNum); } -SearchableFeedView::UpdateScope -SearchableFeedView::getUpdateScope(const DocumentUpdate &upd) -{ - UpdateScope updateScope; - const Schema &schema = *_schema; - for(size_t i(0), m(upd.getUpdates().size()); - !(updateScope._indexedFields && updateScope._nonAttributeFields) && - (i < m); i++) { - const document::FieldUpdate & fu(upd.getUpdates()[i]); - const vespalib::string &name = fu.getField().getName(); - if (schema.isIndexField(name)) { - updateScope._indexedFields = true; - } - if (!fastPartialUpdateAttribute(name)) { - updateScope._nonAttributeFields = true; - } - } - if (!upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; - } - return updateScope; -} - void SearchableFeedView::updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc futureDoc, bool immediateCommit, OnOperationDoneType onWriteDone) diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h index 81d81a6cc27..ed3ba6740b1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h @@ -34,24 +34,19 @@ private: bool hasIndexedFields() const { return _hasIndexedFields; } - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, - bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, + bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &doc, - bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc doc, - bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &doc, + bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc doc, + bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexRemove(SerialNum serialNum, search::DocumentIdT lid, - bool immediateCommit, OnRemoveDoneType onWriteDone); + void performIndexRemove(SerialNum serialNum, search::DocumentIdT lid, + bool immediateCommit, OnRemoveDoneType onWriteDone); - void - performIndexRemove(SerialNum serialNum, const LidVector &lidsToRemove, - bool immediateCommit, OnWriteDoneType onWriteDone); + void performIndexRemove(SerialNum serialNum, const LidVector &lidsToRemove, + bool immediateCommit, OnWriteDoneType onWriteDone); void performIndexHeartBeat(SerialNum serialNum); @@ -60,23 +55,17 @@ private: void performSync(); void heartBeatIndexedFields(SerialNum serialNum) override; - virtual void - putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &newDoc, - bool immediateCommit, OnOperationDoneType onWriteDone) override; + void putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &newDoc, + bool immediateCommit, OnOperationDoneType onWriteDone) override; - UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override; + void updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc newDoc, + bool immediateCommit, OnOperationDoneType onWriteDone) override; - virtual void - updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc newDoc, - bool immediateCommit, OnOperationDoneType onWriteDone) override; + void removeIndexedFields(SerialNum serialNum, search::DocumentIdT lid, + bool immediateCommit, OnRemoveDoneType onWriteDone) override; - virtual void - removeIndexedFields(SerialNum serialNum, search::DocumentIdT lid, - bool immediateCommit, OnRemoveDoneType onWriteDone) override; - - virtual void - removeIndexedFields(SerialNum serialNum, const LidVector &lidsToRemove, - bool immediateCommit, OnWriteDoneType onWriteDone) override; + void removeIndexedFields(SerialNum serialNum, const LidVector &lidsToRemove, + bool immediateCommit, OnWriteDoneType onWriteDone) override; void performIndexForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone); void forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) override; @@ -85,7 +74,7 @@ public: SearchableFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx, const PersistentParams ¶ms, const FastAccessFeedView::Context &fastUpdateCtx, Context ctx); - virtual ~SearchableFeedView(); + ~SearchableFeedView() override; const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; } void sync() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index fdaf07dc466..a0f6ee98b71 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -17,6 +17,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> + LOG_SETUP(".proton.server.storeonlyfeedview"); using document::BucketId; @@ -300,20 +302,19 @@ StoreOnlyFeedView::heartBeatIndexedFields(SerialNum ) {} void StoreOnlyFeedView::heartBeatAttributes(SerialNum ) {} - -StoreOnlyFeedView::UpdateScope -StoreOnlyFeedView::getUpdateScope(const DocumentUpdate &upd) +void +StoreOnlyFeedView::updateAttributes(SerialNum, Lid, const DocumentUpdate & upd, bool, + OnOperationDoneType, IFieldUpdateCallback & onUpdate) { - UpdateScope updateScope; - if (!upd.getUpdates().empty() || !upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; + for (const auto & fieldUpdate : upd.getUpdates()) { + onUpdate.onUpdateField(fieldUpdate.getField().getName(), nullptr); } - return updateScope; } - void -StoreOnlyFeedView::updateAttributes(SerialNum, Lid, const DocumentUpdate &, bool, OnOperationDoneType) {} +StoreOnlyFeedView::updateAttributes(SerialNum, Lid, FutureDoc, bool, OnOperationDoneType) +{ +} void StoreOnlyFeedView::updateIndexedFields(SerialNum, Lid, FutureDoc, bool, OnOperationDoneType) @@ -385,6 +386,34 @@ void StoreOnlyFeedView::heartBeatSummary(SerialNum serialNum) { })); } +StoreOnlyFeedView::UpdateScope::UpdateScope(const search::index::Schema & schema, const DocumentUpdate & upd) + : _schema(&schema), + _indexedFields(false), + _nonAttributeFields(!upd.getFieldPathUpdates().empty()) +{} + +namespace { + +bool isAttributeUpdateable(const search::AttributeVector *attribute) { + search::attribute::BasicType::Type attrType = attribute->getBasicType(); + // Partial update to tensor, predicate or reference attribute + // must update document + return ((attrType != search::attribute::BasicType::Type::PREDICATE) && + (attrType != search::attribute::BasicType::Type::TENSOR) && + (attrType != search::attribute::BasicType::Type::REFERENCE)); +} +} + +void +StoreOnlyFeedView::UpdateScope::onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) { + if (!_nonAttributeFields && (attr == nullptr || !isAttributeUpdateable(attr))) { + _nonAttributeFields = true; + } + if (!_indexedFields && _schema->isIndexField(fieldName)) { + _indexedFields = true; + } +} + void StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) { if ( ! updOp.getUpdate()) { @@ -417,15 +446,15 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) bool immediateCommit = _commitTimeTracker.needCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), updOp.getUpdate()); - updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone); + UpdateScope updateScope(*_schema, upd); + updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); - UpdateScope updateScope(getUpdateScope(upd)); if (updateScope.hasIndexOrNonAttributeFields()) { PromisedDoc promisedDoc; - FutureDoc futureDoc = promisedDoc.get_future(); + FutureDoc futureDoc = promisedDoc.get_future().share(); _pendingLidTracker.waitForConsumedLid(lid); if (updateScope._indexedFields) { - updateIndexedFields(serialNum, lid, std::move(futureDoc), immediateCommit, onWriteDone); + updateIndexedFields(serialNum, lid, futureDoc, immediateCommit, onWriteDone); } PromisedStream promisedStream; FutureStream futureStream = promisedStream.get_future(); @@ -444,6 +473,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) std::move(promisedDoc), std::move(promisedStream)); }); #pragma GCC diagnostic pop + updateAttributes(serialNum, lid, std::move(futureDoc), immediateCommit, onWriteDone); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index b106b87c4fe..a11512590f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -9,6 +9,7 @@ #include "searchcontext.h" #include "pendinglidtracker.h" #include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/common/feeddebugger.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> @@ -33,6 +34,7 @@ class PutDoneContext; class RemoveDoneContext; class CommitTimeTracker; class IGidToLidChangeHandler; +class IFieldUpdateCallback; namespace documentmetastore { class ILidReuseDelayer; } @@ -59,7 +61,7 @@ public: using OnPutDoneType = const std::shared_ptr<PutDoneContext> &; using OnRemoveDoneType = const std::shared_ptr<RemoveDoneContext> &; using FeedTokenUP = std::unique_ptr<FeedToken>; - using FutureDoc = std::future<std::unique_ptr<Document>>; + using FutureDoc = std::shared_future<std::unique_ptr<Document>>; using PromisedDoc = std::promise<std::unique_ptr<Document>>; using FutureStream = std::future<vespalib::nbostream>; using PromisedStream = std::promise<vespalib::nbostream>; @@ -120,18 +122,19 @@ public: }; protected: - struct UpdateScope + class UpdateScope : public IFieldUpdateCallback { + private: + const search::index::Schema *_schema; + public: bool _indexedFields; bool _nonAttributeFields; - UpdateScope() - : _indexedFields(false), - _nonAttributeFields(false) - {} + UpdateScope(const search::index::Schema & schema, const DocumentUpdate & upd); bool hasIndexOrNonAttributeFields() const { return _indexedFields || _nonAttributeFields; } + void onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) override; }; private: @@ -200,9 +203,10 @@ private: virtual void putIndexedFields(SerialNum serialNum, Lid lid, const DocumentSP &newDoc, bool immediateCommit, OnOperationDoneType onWriteDone); - virtual UpdateScope getUpdateScope(const DocumentUpdate &upd); - virtual void updateAttributes(SerialNum serialNum, Lid lid, const DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate); + + virtual void updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone); virtual void updateIndexedFields(SerialNum serialNum, Lid lid, FutureDoc doc, diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h index e0fa06c5e84..a524f2ce0fa 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h @@ -83,6 +83,8 @@ protected: public: AttributeIteratorT(const SC &searchContext, fef::TermFieldMatchData *matchData); bool seekFast(uint32_t docId) const { return _searchContext.matches(docId); } + + const attribute::ISearchContext * getAttributeSearchContext() const override { return &_searchContext; } }; template <typename SC> diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 4dbcc85d861..1ccc9923c99 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -73,7 +73,7 @@ AttributeVector::BaseName::BaseName(const vespalib::stringref &base, append(name); } -AttributeVector::BaseName::~BaseName() { } +AttributeVector::BaseName::~BaseName() = default; AttributeVector::BaseName::string @@ -165,9 +165,7 @@ AttributeVector::AttributeVector(const vespalib::stringref &baseFileName, const _genHandler(), _genHolder(), _status(Status::createName((_baseFileName.getIndexName() + - (_baseFileName.getSnapshotName().empty() ? - "" : - ".") + + (_baseFileName.getSnapshotName().empty() ? "" : ".") + _baseFileName.getSnapshotName()), _baseFileName.getAttributeName())), _highestValueCount(1), @@ -177,29 +175,24 @@ AttributeVector::AttributeVector(const vespalib::stringref &baseFileName, const _createSerialNum(0u), _compactLidSpaceGeneration(0u), _hasEnum(false), - _hasSortedEnum(false), _loaded(false), _enableEnumeratedSave(false) { } - -AttributeVector::~AttributeVector() { } +AttributeVector::~AttributeVector() = default; void AttributeVector::updateStat(bool force) { if (force) { onUpdateStat(); } else if (_nextStatUpdateTime < fastos::ClockSystem::now()) { onUpdateStat(); - _nextStatUpdateTime = fastos::ClockSystem::now() + - fastos::TimeStamp::SEC; + _nextStatUpdateTime = fastos::ClockSystem::now() + fastos::TimeStamp::SEC; } } -size_t AttributeVector::getFixedWidth() const { return _config.basicType().fixedSize(); } bool AttributeVector::hasEnum() const { return _hasEnum; } bool AttributeVector::hasEnum2Value() const { return false; } uint32_t AttributeVector::getMaxValueCount() const { return _highestValueCount; } -uint32_t AttributeVector::getNumDocs() const { return _status.getNumDocs(); } bool AttributeVector::isEnumerated(const vespalib::GenericHeader &header) @@ -217,13 +210,11 @@ AttributeVector::commit(bool forceUpdateStat) _loaded = true; } - void AttributeVector::commit(uint64_t firstSyncToken, uint64_t lastSyncToken) { if (firstSyncToken < getStatus().getLastSyncToken()) { - LOG(error, - "Expected first token to be >= %" PRIu64 ", got %" PRIu64 ".", + LOG(error, "Expected first token to be >= %" PRIu64 ", got %" PRIu64 ".", getStatus().getLastSyncToken(), firstSyncToken); abort(); } @@ -231,7 +222,6 @@ AttributeVector::commit(uint64_t firstSyncToken, uint64_t lastSyncToken) _status.setLastSyncToken(lastSyncToken); } - bool AttributeVector::addDocs(DocId &startDoc, DocId &lastDoc, uint32_t numDocs) { @@ -271,19 +261,10 @@ AttributeVector::incGeneration() void -AttributeVector::updateStatistics(uint64_t numValues, - uint64_t numUniqueValue, - uint64_t allocated, - uint64_t used, - uint64_t dead, - uint64_t onHold) +AttributeVector::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, + uint64_t used, uint64_t dead, uint64_t onHold) { - _status.updateStatistics(numValues, - numUniqueValue, - allocated, - used, - dead, - onHold); + _status.updateStatistics(numValues, numUniqueValue, allocated, used, dead, onHold); } AddressSpace @@ -292,16 +273,6 @@ AttributeVector::getEnumStoreAddressSpaceUsage() const return AddressSpaceUsage::defaultEnumStoreUsage(); } -bool -AttributeVector::hasMultiValue() const { - return _config.collectionType().isMultiValue(); -} - -bool -AttributeVector::hasWeightedSetType() const { - return _config.collectionType().isWeightedSet(); -} - AddressSpace AttributeVector::getMultiValueAddressSpaceUsage() const { @@ -311,38 +282,7 @@ AttributeVector::getMultiValueAddressSpaceUsage() const AddressSpaceUsage AttributeVector::getAddressSpaceUsage() const { - return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), - getMultiValueAddressSpaceUsage()); -} - -const vespalib::string & -AttributeVector::getName() const { - return _baseFileName.getAttributeName(); -} - -attribute::BasicType::Type -AttributeVector::getBasicType() const { - return getInternalBasicType().type(); -} -attribute::CollectionType::Type -AttributeVector::getCollectionType() const { - return getInternalCollectionType().type(); -} - -bool -AttributeVector::getIsFilter() const { - return _config.getIsFilter(); -} - -bool -AttributeVector::getIsFastSearch() const { - return _config.fastSearch(); -} - -uint32_t -AttributeVector::getCommittedDocIdLimit() const -{ - return _committedDocIdLimit; + return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), getMultiValueAddressSpaceUsage()); } bool diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index b25c7b67299..87ef9a41432 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -213,7 +213,6 @@ protected: void setEnumMax(uint32_t e) { _enumMax = e; setEnum(); } void setEnum(bool hasEnum_=true) { _hasEnum = hasEnum_; } - void setSortedEnum(bool sorted=true) { _hasSortedEnum = sorted; } void setNumDocs(uint32_t n) { _status.setNumDocs(n); } void incNumDocs() { _status.incNumDocs(); } @@ -398,7 +397,7 @@ public: bool isLoaded() const { return _loaded; } /** Return the fixed length of the attribute. If 0 then you must inquire each document. */ - virtual size_t getFixedWidth() const override; + size_t getFixedWidth() const override { return _config.basicType().fixedSize(); } const Config &getConfig() const { return _config; } BasicType getInternalBasicType() const { return _config.basicType(); } CollectionType getInternalCollectionType() const { return _config.collectionType(); } @@ -406,17 +405,16 @@ public: void setBaseFileName(const vespalib::stringref & name) { _baseFileName = name; } // Implements IAttributeVector - virtual const vespalib::string & getName() const override; + const vespalib::string & getName() const override final { return _baseFileName.getAttributeName(); } bool hasArrayType() const { return _config.collectionType().isArray(); } - bool hasEnum() const override; - bool hasSortedEnum() const { return _hasSortedEnum; } + bool hasEnum() const override final; virtual bool hasEnum2Value() const; uint32_t getMaxValueCount() const override; uint32_t getEnumMax() const { return _enumMax; } // Implements IAttributeVector - uint32_t getNumDocs() const override; + uint32_t getNumDocs() const override final { return _status.getNumDocs(); } uint32_t & getCommittedDocIdLimitRef() { return _committedDocIdLimit; } void setCommittedDocIdLimit(uint32_t committedDocIdLimit) { _committedDocIdLimit = committedDocIdLimit; @@ -427,13 +425,12 @@ public: AddressSpaceUsage getAddressSpaceUsage() const; - // Implements IAttributeVector - virtual BasicType::Type getBasicType() const override; - virtual CollectionType::Type getCollectionType() const override; - virtual bool getIsFilter() const override; - virtual bool getIsFastSearch() const override; - virtual uint32_t getCommittedDocIdLimit() const override; - virtual bool isImported() const override; + BasicType::Type getBasicType() const override final { return getInternalBasicType().type(); } + CollectionType::Type getCollectionType() const override final { return getInternalCollectionType().type(); } + bool getIsFilter() const override final { return _config.getIsFilter(); } + bool getIsFastSearch() const override final { return _config.fastSearch(); } + uint32_t getCommittedDocIdLimit() const override final { return _committedDocIdLimit; } + bool isImported() const override; /** * Updates the base file name of this attribute vector and saves @@ -604,7 +601,6 @@ private: uint64_t _createSerialNum; uint64_t _compactLidSpaceGeneration; bool _hasEnum; - bool _hasSortedEnum; bool _loaded; bool _enableEnumeratedSave; fastos::TimeStamp _nextStatUpdateTime; @@ -634,8 +630,8 @@ private: friend class AttributeManagerTest; public: bool headerTypeOK(const vespalib::GenericHeader &header) const; - bool hasMultiValue() const override; - bool hasWeightedSetType() const override; + bool hasMultiValue() const override final { return _config.collectionType().isMultiValue(); } + bool hasWeightedSetType() const override final { return _config.collectionType().isWeightedSet(); } /** * Should be called by the writer thread. */ diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp index abcdc0244c2..8f67d237a60 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp @@ -170,7 +170,6 @@ StringDirectAttrVector(const vespalib::string & baseFileName, const Config & c) _idx.push_back(0); } setEnum(); - setSortedEnum(true); } template <typename F> @@ -182,6 +181,5 @@ StringDirectAttrVector(const vespalib::string & baseFileName) : _idx.push_back(0); } setEnum(); - setSortedEnum(true); } diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp index d4fa47eac49..ee984688594 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp @@ -12,7 +12,6 @@ SingleStringExtAttribute::SingleStringExtAttribute(const vespalib::string & name StringDirectAttrVector< AttrVector::Features<false> >(name, Config(BasicType::STRING, CollectionType::SINGLE)) { setEnum(false); - setSortedEnum(false); } bool SingleStringExtAttribute::addDoc(DocId & docId) @@ -45,7 +44,6 @@ MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name, (name, Config(BasicType::STRING, ctype)) { setEnum(false); - setSortedEnum(false); } MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name) : @@ -53,7 +51,6 @@ MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name) (name, Config(BasicType::STRING, CollectionType::ARRAY)) { setEnum(false); - setSortedEnum(false); } bool MultiStringExtAttribute::addDoc(DocId & docId) @@ -138,7 +135,6 @@ WeightedSetStringExtAttribute::WeightedSetStringExtAttribute(const vespalib::str WeightedSetExtAttributeBase<MultiStringExtAttribute>(name) { setEnum(false); - setSortedEnum(false); } WeightedSetStringExtAttribute::~WeightedSetStringExtAttribute() {} diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index a563a288396..840bb1e5ea9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -3,6 +3,7 @@ #include "same_element_blueprint.h" #include "same_element_search.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/attribute/elementiterator.h> #include <vespa/vespalib/objects/visit.hpp> #include <algorithm> #include <map> @@ -60,14 +61,29 @@ SameElementBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArra { (void) tfmda; assert(!tfmda.valid()); - fef::MatchData::UP md = _layout.createMatchData(); + + fef::MatchDataLayout my_layout = _layout; + std::vector<fef::TermFieldHandle> extra_handles; + for (size_t i = 0; i < _terms.size(); ++i) { + const State &childState = _terms[i]->getState(); + assert(childState.numFields() == 1); + extra_handles.push_back(my_layout.allocTermField(childState.field(0).getFieldId())); + } + fef::MatchData::UP md = my_layout.createMatchData(); search::fef::TermFieldMatchDataArray childMatch; std::vector<SearchIterator::UP> children(_terms.size()); for (size_t i = 0; i < _terms.size(); ++i) { const State &childState = _terms[i]->getState(); - assert(childState.numFields() == 1); - childMatch.add(childState.field(0).resolve(*md)); - children[i] = _terms[i]->createSearch(*md, (strict && (i == 0))); + SearchIterator::UP child = _terms[i]->createSearch(*md, (strict && (i == 0))); + const attribute::ISearchContext *context = child->getAttributeSearchContext(); + if (context == nullptr) { + children[i] = std::move(child); + childMatch.add(childState.field(0).resolve(*md)); + } else { + fef::TermFieldMatchData *child_tfmd = md->resolveTermField(extra_handles[i]); + children[i] = std::make_unique<attribute::ElementIterator>(std::move(child), *context, *child_tfmd); + childMatch.add(child_tfmd); + } } return std::make_unique<SameElementSearch>(std::move(md), std::move(children), childMatch, strict); } diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp index 1f5d090b914..3384e0fc8c8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp @@ -118,7 +118,13 @@ SearchIterator::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "docid", _docid); visit(visitor, "endid", _endid); } - + +const attribute::ISearchContext * +SearchIterator::getAttributeSearchContext() const +{ + return nullptr; +} + } // namespace queryeval } // namespace search diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h index 63c2afd33aa..dfa342b018a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h @@ -12,6 +12,7 @@ namespace vespalib { class ObjectVisitor; } namespace search { class BitVector; } +namespace search::attribute { class ISearchContext; } namespace search::queryeval { @@ -338,6 +339,8 @@ public: virtual Trinary is_strict() const { return Trinary::Undefined; } + /** return the underlying attribute search context (or null if none available) */ + virtual const attribute::ISearchContext *getAttributeSearchContext() const; }; } diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 0d66951f364..f31072aec6e 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -69,11 +69,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-xml_${scala.major-version}</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> @@ -101,40 +96,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> </plugin> - - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <id>compile</id> - <goals> - <goal>compile</goal> - </goals> - <phase>compile</phase> - </execution> - <execution> - <id>test-compile</id> - <goals> - <goal>testCompile</goal> - </goals> - <phase>test-compile</phase> - </execution> - <execution> - <phase>process-resources</phase> - <goals> - <goal>compile</goal> - </goals> - </execution> - <execution> - <phase>process-test-resources</phase> - <id>early-test-compile</id> - <goals> - <goal>testCompile</goal> - </goals> - </execution> - </executions> - </plugin> </plugins> </build> </project> diff --git a/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java b/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java new file mode 100644 index 00000000000..8d4126a01e3 --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java @@ -0,0 +1,569 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.impl; + +import com.google.common.collect.Lists; +import com.yahoo.container.standalone.StandaloneContainerApplication; +import com.yahoo.jdisc.application.OsgiFramework; +import com.yahoo.jdisc.application.OsgiHeader; +import org.osgi.framework.Bundle; +import org.osgi.framework.BundleContext; +import org.osgi.framework.BundleListener; +import org.osgi.framework.Filter; +import org.osgi.framework.FrameworkListener; +import org.osgi.framework.ServiceFactory; +import org.osgi.framework.ServiceListener; +import org.osgi.framework.ServiceObjects; +import org.osgi.framework.ServiceReference; +import org.osgi.framework.ServiceRegistration; +import org.osgi.framework.Version; +import org.osgi.framework.wiring.BundleCapability; +import org.osgi.framework.wiring.BundleRequirement; +import org.osgi.framework.wiring.BundleRevision; +import org.osgi.framework.wiring.BundleWire; +import org.osgi.framework.wiring.BundleWiring; +import org.osgi.resource.Capability; +import org.osgi.resource.Requirement; +import org.osgi.resource.Wire; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Dictionary; +import java.util.Enumeration; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.jar.Attributes; +import java.util.jar.JarFile; + +/** + * A (mock) OSGI implementation which loads classes from the system classpath + * + * @author Tony Vaagenes + * @author ollivir + */ +public final class ClassLoaderOsgiFramework implements OsgiFramework { + private BundleContextImpl bundleContextImpl = new BundleContextImpl(); + private SystemBundleImpl systemBundleImpl = new SystemBundleImpl(); + private BundleWiringImpl bundleWiringImpl = new BundleWiringImpl(); + + private List<URL> bundleLocations = new ArrayList<>(); + private List<Bundle> bundleList = Lists.newArrayList(systemBundleImpl); + private ClassLoader classLoader = null; + + private AtomicInteger nextBundleId = new AtomicInteger(1); + + @Override + public List<Bundle> installBundle(String bundleLocation) { + if (bundleLocation != null && bundleLocation.isEmpty() == false) { + try { + URL url = new URL(bundleLocation); + bundleLocations.add(url); + bundleList.add(new JarBundleImpl(url)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return bundles(); + } + + private ClassLoader getClassLoader() { + if (bundleLocations.isEmpty()) { + return getClass().getClassLoader(); + } else { + if (classLoader == null) { + classLoader = new URLClassLoader(bundleLocations.toArray(new URL[0]), getClass().getClassLoader()); + } + return classLoader; + } + } + + @Override + public void startBundles(List<Bundle> bundles, boolean privileged) { + } + + @Override + public void refreshPackages() { + } + + @Override + public BundleContext bundleContext() { + return bundleContextImpl; + } + + @Override + public List<Bundle> bundles() { + return bundleList; + } + + @Override + public void start() { + } + + @Override + public void stop() { + } + + private abstract class BundleImpl implements Bundle { + @Override + public int getState() { + return Bundle.ACTIVE; + } + + @Override + public void start(int options) { + } + + @Override + public void start() { + } + + @Override + public void stop(int options) { + } + + @Override + public void stop() { + } + + @Override + public void update(InputStream input) { + } + + @Override + public void update() { + } + + @Override + public void uninstall() { + } + + @Override + public Dictionary<String, String> getHeaders(String locale) { + return getHeaders(); + } + + @Override + public String getSymbolicName() { + return ClassLoaderOsgiFramework.this.getClass().getName(); + } + + @Override + public String getLocation() { + return getSymbolicName(); + } + + @Override + public ServiceReference<?>[] getRegisteredServices() { + return new ServiceReference<?>[0]; + } + + @Override + public ServiceReference<?>[] getServicesInUse() { + return getRegisteredServices(); + } + + @Override + public boolean hasPermission(Object permission) { + return true; + } + + @Override + public URL getResource(String name) { + return getClassLoader().getResource(name); + } + + @Override + public Class<?> loadClass(String name) throws ClassNotFoundException { + return getClassLoader().loadClass(name); + } + + @Override + public Enumeration<URL> getResources(String name) throws IOException { + return getClassLoader().getResources(name); + } + + @Override + public Enumeration<String> getEntryPaths(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public URL getEntry(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public Enumeration<URL> findEntries(String path, String filePattern, boolean recurse) { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastModified() { + return 1L; + } + + @Override + public BundleContext getBundleContext() { + throw new UnsupportedOperationException(); + } + + @Override + public Map<X509Certificate, List<X509Certificate>> getSignerCertificates(int signersType) { + return Collections.emptyMap(); + } + + @Override + @SuppressWarnings("unchecked") + public <T> T adapt(Class<T> clazz) { + if (clazz.equals(BundleRevision.class)) { + return (T) new BundleRevisionImpl(); + } else if (clazz.equals(BundleWiring.class)) { + return (T) new BundleWiringImpl(); + } else { + return null; + } + } + + @Override + public File getDataFile(String filename) { + return null; + } + + @Override + public int compareTo(Bundle o) { + return Long.compare(getBundleId(), o.getBundleId()); + } + } + + private class BundleRevisionImpl implements BundleRevision { + @Override + public String getSymbolicName() { + return this.getClass().getName(); + } + + @Override + public List<BundleRequirement> getDeclaredRequirements(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public Version getVersion() { + return Version.emptyVersion; + } + + @Override + public BundleWiring getWiring() { + return bundleWiringImpl; + } + + @Override + public List<BundleCapability> getDeclaredCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public int getTypes() { + return 0; + } + + @Override + public Bundle getBundle() { + throw new UnsupportedOperationException(); + } + + @Override + public List<Capability> getCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Requirement> getRequirements(String p1) { + throw new UnsupportedOperationException(); + } + } + + private class BundleWiringImpl implements BundleWiring { + @Override + public List<URL> findEntries(String p1, String p2, int p3) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Wire> getRequiredResourceWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Capability> getResourceCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCurrent() { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleWire> getRequiredWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleCapability> getCapabilities(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<Wire> getProvidedResourceWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public List<BundleWire> getProvidedWires(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public BundleRevision getRevision() { + throw new UnsupportedOperationException(); + } + + @Override + public List<Requirement> getResourceRequirements(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isInUse() { + throw new UnsupportedOperationException(); + } + + @Override + public Collection<String> listResources(String p1, String p2, int p3) { + throw new UnsupportedOperationException(); + } + + @Override + public ClassLoader getClassLoader() { + return ClassLoaderOsgiFramework.this.getClassLoader(); + } + + @Override + public List<BundleRequirement> getRequirements(String p1) { + throw new UnsupportedOperationException(); + } + + @Override + public BundleRevision getResource() { + throw new UnsupportedOperationException(); + } + + @Override + public Bundle getBundle() { + throw new UnsupportedOperationException(); + } + } + + private class SystemBundleImpl extends BundleImpl { + @Override + public long getBundleId() { + return 0L; + } + + @Override + public Version getVersion() { + return Version.emptyVersion; + } + + @Override + public Dictionary<String, String> getHeaders() { + Hashtable<String, String> ret = new Hashtable<>(); + ret.put(OsgiHeader.APPLICATION, StandaloneContainerApplication.class.getName()); + return ret; + } + } + + private class JarBundleImpl extends BundleImpl { + private final long bundleId; + private final Dictionary<String, String> headers; + + JarBundleImpl(URL location) throws IOException { + this.bundleId = (long) nextBundleId.getAndIncrement(); + this.headers = retrieveHeaders(location); + } + + @Override + public long getBundleId() { + return bundleId; + } + + @Override + public Dictionary<String, String> getHeaders() { + return headers; + } + + @Override + public String getSymbolicName() { + return headers.get("Bundle-SymbolicName"); + } + + @Override + public Version getVersion() { + return Version.parseVersion(headers.get("Bundle-Version")); + } + + private Dictionary<String, String> retrieveHeaders(URL location) throws IOException { + try (JarFile jarFile = new JarFile(location.getFile())) { + Attributes attributes = jarFile.getManifest().getMainAttributes(); + Hashtable<String, String> ret = new Hashtable<>(); + attributes.forEach((k, v) -> ret.put(k.toString(), v.toString())); + return ret; + } + } + } + + private class BundleContextImpl implements BundleContext { + @Override + public String getProperty(String key) { + return null; + } + + @Override + public Bundle getBundle() { + return systemBundleImpl; + } + + @Override + public Bundle installBundle(String location, InputStream input) { + throw new UnsupportedOperationException(); + } + + @Override + public Bundle installBundle(String location) { + throw new UnsupportedOperationException(); + } + + @Override + public Bundle getBundle(long id) { + return systemBundleImpl; + } + + @Override + public Bundle[] getBundles() { + return new Bundle[] { systemBundleImpl }; + } + + @Override + public Bundle getBundle(String location) { + return systemBundleImpl; + } + + @Override + public void addServiceListener(ServiceListener listener, String filter) { + } + + @Override + public void addServiceListener(ServiceListener listener) { + } + + @Override + public void removeServiceListener(ServiceListener listener) { + } + + @Override + public void addBundleListener(BundleListener listener) { + } + + @Override + public void removeBundleListener(BundleListener listener) { + } + + @Override + public void addFrameworkListener(FrameworkListener listener) { + } + + @Override + public void removeFrameworkListener(FrameworkListener listener) { + } + + @Override + public ServiceRegistration<?> registerService(String[] classes, Object service, Dictionary<String, ?> properties) { + throw new UnsupportedOperationException(); + } + + @Override + public ServiceRegistration<?> registerService(String clazz, Object service, Dictionary<String, ?> properties) { + return null; + } + + @Override + public <S> ServiceRegistration<S> registerService(Class<S> clazz, S service, Dictionary<String, ?> properties) { + throw new UnsupportedOperationException(); + } + + @Override + public ServiceReference<?>[] getServiceReferences(String clazz, String filter) { + throw new UnsupportedOperationException(); + } + + @Override + public ServiceReference<?>[] getAllServiceReferences(String clazz, String filter) { + throw new UnsupportedOperationException(); + } + + @Override + public ServiceReference<?> getServiceReference(String clazz) { + throw new UnsupportedOperationException(); + } + + @Override + public <S> ServiceReference<S> getServiceReference(Class<S> clazz) { + throw new UnsupportedOperationException(); + } + + @Override + public <S> Collection<ServiceReference<S>> getServiceReferences(Class<S> clazz, String filter) { + return new ArrayList<>(); + } + + @Override + public <S> S getService(ServiceReference<S> reference) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean ungetService(ServiceReference<?> reference) { + throw new UnsupportedOperationException(); + } + + @Override + public File getDataFile(String filename) { + throw new UnsupportedOperationException(); + } + + @Override + public Filter createFilter(String filter) { + throw new UnsupportedOperationException(); + } + + @Override + public <S> ServiceRegistration<S> registerService(Class<S> aClass, ServiceFactory<S> serviceFactory, + Dictionary<String, ?> dictionary) { + throw new UnsupportedOperationException(); + } + + @Override + public <S> ServiceObjects<S> getServiceObjects(ServiceReference<S> serviceReference) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java b/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java new file mode 100644 index 00000000000..a0fee3265df --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/application/container/impl/StandaloneContainerRunner.java @@ -0,0 +1,34 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.impl; + +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class StandaloneContainerRunner { + public static Path createApplicationPackage(String servicesXml) { + try { + return createApplicationDirectory(servicesXml); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static Path createApplicationDirectory(String servicesXml) throws IOException { + Path applicationDir = Files.createTempDirectory("application"); + Path servicesXmlFile = applicationDir.resolve("services.xml"); + String content = servicesXml; + + if (!servicesXml.startsWith("<?xml")) { + content = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + servicesXml; + } + Files.write(servicesXmlFile, Utf8.toBytes(content)); + return applicationDir; + } +} diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java new file mode 100644 index 00000000000..4bbe9986d90 --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/LocalFileDb.java @@ -0,0 +1,97 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.net.HostName; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * FileAcquirer and FileRegistry working on a local directory. + * + * @author Tony Vaagenes + * @author ollivir + */ +public class LocalFileDb implements FileAcquirer, FileRegistry { + private static final Constructor<FileReference> fileReferenceConstructor = createFileReferenceConstructor(); + + private final Map<FileReference, File> fileReferenceToFile = new HashMap<>(); + private final Path appPath; + + public LocalFileDb(Path appPath) { + this.appPath = appPath; + } + + /* FileAcquirer overrides */ + @Override + public File waitFor(FileReference reference, long l, TimeUnit timeUnit) { + synchronized (this) { + File file = fileReferenceToFile.get(reference); + if (file == null) { + throw new RuntimeException("Invalid file reference " + reference); + } + return file; + } + } + + @Override + public void shutdown() { + } + + /* FileRegistry overrides */ + public FileReference addFile(String relativePath) { + File file = appPath.resolve(relativePath).toFile(); + if (!file.exists()) { + throw new RuntimeException("The file does not exist: " + file.getPath()); + } + + FileReference fileReference = null; + try { + fileReference = fileReferenceConstructor.newInstance("LocalFileDb:" + relativePath); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException("Unable to create new FileReference", e); + } + fileReferenceToFile.put(fileReference, file); + return fileReference; + } + + @Override + public List<Entry> export() { + return fileReferenceToFile.entrySet().stream().map(entry -> new Entry(entry.getValue().getPath(), entry.getKey())) + .collect(Collectors.toList()); + } + + @Override + public FileReference addUri(String uri) { + throw new RuntimeException("addUri(String uri) is not implemented here."); + } + + public String fileSourceHost() { + return HostName.getLocalhost(); + } + + public Set<String> allRelativePaths() { + return fileReferenceToFile.values().stream().map(File::getPath).collect(Collectors.toSet()); + } + + private static Constructor<FileReference> createFileReferenceConstructor() { + try { + Constructor<FileReference> method = FileReference.class.getDeclaredConstructor(String.class); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ex) { + throw new IllegalStateException(ex); + } + } +} diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java new file mode 100644 index 00000000000..72937301954 --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java @@ -0,0 +1,304 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.google.inject.AbstractModule; +import com.google.inject.ConfigurationException; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.ProvisionException; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.di.config.SubscriberFactory; +import com.yahoo.container.jdisc.ConfiguredApplication; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.application.Application; +import com.yahoo.text.XML; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.xml.ConfigServerContainerModelBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; +import org.w3c.dom.Element; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.yahoo.collections.CollectionUtil.first; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public class StandaloneContainerApplication implements Application { + public static final String PACKAGE_NAME = "standalone_jdisc_container"; + public static final String APPLICATION_LOCATION_INSTALL_VARIABLE = PACKAGE_NAME + ".app_location"; + public static final String DEPLOYMENT_PROFILE_INSTALL_VARIABLE = PACKAGE_NAME + ".deployment_profile"; + public static final String DISABLE_NETWORKING_ANNOTATION = "JDisc.disableNetworking"; + public static final Named APPLICATION_PATH_NAME = Names.named(APPLICATION_LOCATION_INSTALL_VARIABLE); + public static final Named CONFIG_MODEL_REPO_NAME = Names.named("ConfigModelRepo"); + + private static final String DEFAULT_TMP_BASE_DIR = Defaults.getDefaults().underVespaHome("tmp"); + private static final String TMP_DIR_NAME = "standalone_container"; + + private static final StaticConfigDefinitionRepo configDefinitionRepo = new StaticConfigDefinitionRepo(); + + private final Injector injector; + private final Path applicationPath; + private final LocalFileDb distributedFiles; + private final ConfigModelRepo configModelRepo; + private final Networking networkingOption; + private final VespaModel modelRoot; + private final Application configuredApplication; + private final Container container; + + @Inject + public StandaloneContainerApplication(Injector injector) { + this.injector = injector; + ConfiguredApplication.ensureVespaLoggingInitialized(); + this.applicationPath = injectedApplicationPath().orElseGet(this::installApplicationPath); + this.distributedFiles = new LocalFileDb(applicationPath); + this.configModelRepo = resolveConfigModelRepo(); + this.networkingOption = resolveNetworkingOption(); + + try { + Pair<VespaModel, Container> tpl = withTempDir(preprocessedApplicationDir -> createContainerModel(applicationPath, + distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo)); + this.modelRoot = tpl.getFirst(); + this.container = tpl.getSecond(); + } catch (RuntimeException r) { + throw r; + } catch (Exception e) { + throw new RuntimeException("Failed to create ContainerModel", e); + } + this.configuredApplication = createConfiguredApplication(container); + } + + private ConfigModelRepo resolveConfigModelRepo() { + try { + return injector.getInstance(Key.get(ConfigModelRepo.class, CONFIG_MODEL_REPO_NAME)); + } catch (Exception e) { + return new ConfigModelRepo(); + } + } + + private Networking resolveNetworkingOption() { + try { + Boolean networkingDisable = injector.getInstance(Key.get(Boolean.class, Names.named(DISABLE_NETWORKING_ANNOTATION))); + if (networkingDisable != null) { + return networkingDisable ? Networking.disable : Networking.enable; + } + } catch (Exception ignored) { + } + return Networking.enable; + } + + private Application createConfiguredApplication(Container container) { + Injector augmentedInjector = injector.createChildInjector(new AbstractModule() { + @Override + public void configure() { + bind(SubscriberFactory.class).toInstance(new StandaloneSubscriberFactory(modelRoot)); + } + }); + + System.setProperty("config.id", container.getConfigId()); + return augmentedInjector.getInstance(ConfiguredApplication.class); + } + + private Optional<Path> injectedApplicationPath() { + try { + return Optional.ofNullable(injector.getInstance(Key.get(Path.class, APPLICATION_PATH_NAME))); + } catch (ConfigurationException | ProvisionException ignored) { + } + return Optional.empty(); + } + + private Path installApplicationPath() { + Optional<String> variable = optionalInstallVariable(APPLICATION_LOCATION_INSTALL_VARIABLE); + + return variable.map(Paths::get) + .orElseThrow(() -> new IllegalStateException("Environment variable not set: " + APPLICATION_LOCATION_INSTALL_VARIABLE)); + } + + @Override + public void start() { + try { + com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles); + configuredApplication.start(); + } catch (Exception e) { + com.yahoo.container.Container.resetInstance(); + throw e; + } + } + + @Override + public void stop() { + configuredApplication.stop(); + } + + @Override + public void destroy() { + com.yahoo.container.Container.resetInstance(); + configuredApplication.destroy(); + } + + public Container container() { + return container; + } + + private interface ThrowingFunction<T, U> { + U apply(T input) throws Exception; + } + + private static <T> T withTempDir(ThrowingFunction<File, T> f) throws Exception { + File tmpDir = createTempDir(); + try { + return f.apply(tmpDir); + } finally { + IOUtils.recursiveDeleteDir(tmpDir); + } + } + + private static File createTempDir() { + Path basePath; + if (new File(DEFAULT_TMP_BASE_DIR).exists()) { + basePath = Paths.get(DEFAULT_TMP_BASE_DIR); + } else { + basePath = Paths.get(System.getProperty("java.io.tmpdir")); + } + + try { + Path tmpDir = Files.createTempDirectory(basePath, TMP_DIR_NAME); + return tmpDir.toFile(); + } catch (IOException e) { + throw new RuntimeException("Cannot create temp directory", e); + } + } + + private static void validateApplication(ApplicationPackage applicationPackage) { + try { + applicationPackage.validateXML(); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private static ContainerModelBuilder newContainerModelBuilder(Networking networkingOption) { + Optional<String> profile = optionalInstallVariable(DEPLOYMENT_PROFILE_INSTALL_VARIABLE); + if (profile.isPresent()) { + String profileName = profile.get(); + if ("configserver".equals(profileName)) { + return new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables()); + } else { + throw new RuntimeException("Invalid deployment profile '" + profileName + "'"); + } + } else { + return new ContainerModelBuilder(true, networkingOption); + } + } + + static Pair<VespaModel, Container> createContainerModel(Path applicationPath, FileRegistry fileRegistry, + File preprocessedApplicationDir, Networking networkingOption, ConfigModelRepo configModelRepo) throws Exception { + DeployLogger logger = new BaseDeployLogger(); + FilesApplicationPackage rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile()) + .includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build(); + ApplicationPackage applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), logger); + validateApplication(applicationPackage); + DeployState deployState = new DeployState.Builder().applicationPackage(applicationPackage).fileRegistry(fileRegistry) + .deployLogger(logger).configDefinitionRepo(configDefinitionRepo).build(true); + + VespaModel root = VespaModel.createIncomplete(deployState); + ApplicationConfigProducerRoot vespaRoot = new ApplicationConfigProducerRoot(root, "vespa", deployState.getDocumentModel(), + deployState.getProperties().vespaVersion(), deployState.getProperties().applicationId()); + + Element spec = containerRootElement(applicationPackage); + ContainerModel containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec); + containerModel.getCluster().prepare(); + initializeContainerModel(containerModel, configModelRepo); + Container container = first(containerModel.getCluster().getContainers()); + + // TODO: Separate out model finalization from the VespaModel constructor, + // such that the above and below code to finalize the container can be + // replaced by root.finalize(); + + initializeContainer(container, spec); + + root.freezeModelTopology(); + return new Pair<>(root, container); + } + + private static void initializeContainer(Container container, Element spec) { + HostResource host = container.getRoot().getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); + + container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec)); + container.setHostResource(host); + container.initService(); + } + + private static Element getJDiscInServices(Element element) { + List<Element> jDiscElements = new ArrayList<>(); + for (ConfigModelId cid : ContainerModelBuilder.configModelIds) { + List<Element> children = XML.getChildren(element, cid.getName()); + jDiscElements.addAll(children); + } + + if (jDiscElements.size() == 1) { + return jDiscElements.get(0); + } else if (jDiscElements.isEmpty()) { + throw new RuntimeException("No jdisc element found under services."); + } else { + List<String> nameAndId = jDiscElements.stream().map(e -> e.getNodeName() + " id='" + e.getAttribute("id") + "'") + .collect(Collectors.toList()); + throw new RuntimeException("Found multiple JDisc elements: " + String.join(", ", nameAndId)); + } + } + + private static Element containerRootElement(ApplicationPackage applicationPackage) { + Element element = XmlHelper.getDocument(applicationPackage.getServices()).getDocumentElement(); + String nodeName = element.getNodeName(); + + if (ContainerModelBuilder.configModelIds.stream().anyMatch(id -> id.getName().equals(nodeName))) { + return element; + } else { + return getJDiscInServices(element); + } + } + + @SuppressWarnings("deprecation") // TODO: what is the not-deprecated way? + private static void initializeContainerModel(ContainerModel containerModel, ConfigModelRepo configModelRepo) { + containerModel.initialize(configModelRepo); + } + + private static Optional<String> optionalInstallVariable(String name) { + Optional<String> fromEnv = Optional.ofNullable(System.getenv((name.replace(".", "__")))); + if (fromEnv.isPresent()) { + return fromEnv; + } + return Optional.ofNullable(System.getProperty(name)); // for unit testing + } +} diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java new file mode 100644 index 00000000000..6ea2671b05b --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java @@ -0,0 +1,131 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.container.di.config.Subscriber; +import com.yahoo.container.di.config.SubscriberFactory; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.model.VespaModel; + +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ +public class StandaloneSubscriberFactory implements SubscriberFactory { + private final VespaModel root; + + public StandaloneSubscriberFactory(VespaModel root) { + this.root = root; + } + + private class StandaloneSubscriber implements Subscriber { + + private final Set<ConfigKey<ConfigInstance>> configKeys; + private long generation = -1L; + + StandaloneSubscriber(Set<ConfigKey<ConfigInstance>> configKeys) { + this.configKeys = configKeys; + } + + @Override + public boolean internalRedeploy() { return false; } + + @Override + public boolean configChanged() { + return generation == 0; + } + + @Override + public void close() { + } + + @Override + public Map<ConfigKey<ConfigInstance>, ConfigInstance> config() { + Map<ConfigKey<ConfigInstance>, ConfigInstance> ret = new HashMap<>(); + for (ConfigKey<ConfigInstance> key : configKeys) { + ConfigInstance.Builder builder = root.getConfig(newBuilderInstance(key), key.getConfigId()); + if (builder == null) { + throw new RuntimeException("Invalid config id " + key.getConfigId()); + } + ret.put(key, newConfigInstance(builder)); + } + return ret; + } + + @Override + public long waitNextGeneration() { + generation++; + + if (generation != 0) { + try { + while (!Thread.interrupted()) { + Thread.sleep(10000); + } + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + } + + return generation; + } + + // if waitNextGeneration has not yet been called, -1 should be returned + @Override + public long generation() { + return generation; + } + } + + @Override + @SuppressWarnings("unchecked") + public Subscriber getSubscriber(Set<? extends ConfigKey<?>> configKeys) { + return new StandaloneSubscriber((Set<ConfigKey<ConfigInstance>>) configKeys); + } + + public void reloadActiveSubscribers(long generation) { + throw new RuntimeException("unsupported"); + } + + private static ConfigInstance.Builder newBuilderInstance(ConfigKey<ConfigInstance> key) { + try { + return builderClass(key).getDeclaredConstructor().newInstance(); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException("ConfigInstance builder cannot be instantiated", e); + } + } + + @SuppressWarnings("unchecked") + private static Class<ConfigInstance.Builder> builderClass(ConfigKey<ConfigInstance> key) { + Class<?> configClass = key.getConfigClass(); + if (configClass != null) { + Class<?>[] nestedClasses = configClass.getClasses(); + for (Class<?> clazz : nestedClasses) { + if (clazz.getName().equals(key.getConfigClass().getName() + "$Builder")) { + return (Class<ConfigInstance.Builder>) clazz; + } + } + } + throw new RuntimeException("Builder class for " + (configClass == null ? null : configClass.getName()) + " could not be located"); + } + + private static ConfigInstance newConfigInstance(ConfigBuilder builder) { + try { + return configClass(builder).getConstructor(builder.getClass()).newInstance(builder); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException("ConfigInstance cannot be instantiated", e); + } + } + + @SuppressWarnings("unchecked") + private static Class<ConfigInstance> configClass(ConfigBuilder builder) { + return (Class<ConfigInstance>) builder.getClass().getEnclosingClass(); + } +} diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala deleted file mode 100644 index ac8636de2cb..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.impl - -import java.io.InputStream -import java.net.{URL, URLClassLoader} -import java.util -import java.util.concurrent.atomic.AtomicInteger -import java.util.jar.JarFile -import java.util.{Collections, Dictionary, Hashtable} - -import com.yahoo.container.standalone.StandaloneContainerApplication -import com.yahoo.jdisc.application.{OsgiFramework, OsgiHeader} -import org.osgi.framework._ -import org.osgi.framework.wiring._ -import org.osgi.resource.{Capability, Requirement, Wire} - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer - -/** - * A (mock) OSGI implementation which loads classes from the system classpath - * - * @author tonytv - */ -final class ClassLoaderOsgiFramework extends OsgiFramework { - private val bundleLocations = new ArrayBuffer[URL] - private val bundleList = ArrayBuffer[Bundle](SystemBundleImpl) - private var classLoader: ClassLoader = null - - private val nextBundleId = new AtomicInteger(1) - - override def installBundle(bundleLocation: String) = { - if (bundleLocation != "") { - val url = new URL(bundleLocation) - bundleLocations += url - bundleList += new JarBundleImpl(url) - } - - bundles() - } - - def getClassLoader = { - if (bundleLocations.isEmpty) { - getClass.getClassLoader - } else { - if(classLoader == null) - classLoader = new URLClassLoader(bundleLocations.toArray, getClass.getClassLoader) - - classLoader - } - } - - override def startBundles(bundles: util.List[Bundle], privileged: Boolean) {} - - override def refreshPackages() {} - - override def bundleContext():BundleContext = BundleContextImpl - - override def bundles() = bundleList.asJava - - override def start() {} - - override def stop() {} - - private abstract class BundleImpl extends Bundle { - override def getState = Bundle.ACTIVE - - override def start(options: Int) {} - override def start() {} - override def stop(options: Int) {} - override def stop() {} - override def update(input: InputStream) {} - override def update() {} - override def uninstall() {} - - override def getHeaders(locale: String) = getHeaders - - override def getSymbolicName = ClassLoaderOsgiFramework.this.getClass.getName - override def getLocation = getSymbolicName - - override def getRegisteredServices = Array[ServiceReference[_]]() - override def getServicesInUse = getRegisteredServices - - override def hasPermission(permission: Any) = true - - override def getResource(name: String) = getClassLoader.getResource(name) - override def loadClass(name: String) = getClassLoader.loadClass(name) - override def getResources(name: String) = getClassLoader.getResources(name) - - override def getEntryPaths(path: String) = throw new UnsupportedOperationException - override def getEntry(path: String) = throw new UnsupportedOperationException - override def findEntries(path: String, filePattern: String, recurse: Boolean) = throw new UnsupportedOperationException - - override def getLastModified = 1L - - override def getBundleContext = throw new UnsupportedOperationException - override def getSignerCertificates(signersType: Int) = Collections.emptyMap() - - override def adapt[A](`type`: Class[A]): A = { - if (`type` == classOf[BundleRevision]) BundleRevisionImpl.asInstanceOf[A] - else if (`type` == classOf[BundleWiring]) BundleWiringImpl.asInstanceOf[A] - else null.asInstanceOf[A] - } - - override def getDataFile(filename: String) = null - override def compareTo(o: Bundle) = getBundleId compareTo o.getBundleId - } - - private object BundleRevisionImpl extends BundleRevision { - override def getSymbolicName: String = this.getClass.getName - override def getDeclaredRequirements(p1: String): util.List[BundleRequirement] = throw new UnsupportedOperationException - override def getVersion: Version = Version.emptyVersion - override def getWiring: BundleWiring = BundleWiringImpl - override def getDeclaredCapabilities(p1: String): util.List[BundleCapability] = throw new UnsupportedOperationException - override def getTypes: Int = 0 - override def getBundle: Bundle = throw new UnsupportedOperationException - override def getCapabilities(p1: String): util.List[Capability] = throw new UnsupportedOperationException - override def getRequirements(p1: String): util.List[Requirement] = throw new UnsupportedOperationException - } - - private object BundleWiringImpl extends BundleWiring { - override def findEntries(p1: String, p2: String, p3: Int): util.List[URL] = ??? - override def getRequiredResourceWires(p1: String): util.List[Wire] = ??? - override def getResourceCapabilities(p1: String): util.List[Capability] = ??? - override def isCurrent: Boolean = ??? - override def getRequiredWires(p1: String): util.List[BundleWire] = ??? - override def getCapabilities(p1: String): util.List[BundleCapability] = ??? - override def getProvidedResourceWires(p1: String): util.List[Wire] = ??? - override def getProvidedWires(p1: String): util.List[BundleWire] = ??? - override def getRevision: BundleRevision = ??? - override def getResourceRequirements(p1: String): util.List[Requirement] = ??? - override def isInUse: Boolean = ??? - override def listResources(p1: String, p2: String, p3: Int): util.Collection[String] = ??? - override def getClassLoader: ClassLoader = ClassLoaderOsgiFramework.this.getClassLoader - override def getRequirements(p1: String): util.List[BundleRequirement] = ??? - override def getResource: BundleRevision = ??? - override def getBundle: Bundle = ??? - } - - private object SystemBundleImpl extends BundleImpl { - override val getBundleId = 0L - override def getVersion = Version.emptyVersion - override def getHeaders: Dictionary[String, String] = new Hashtable[String, String](Map(OsgiHeader.APPLICATION -> classOf[StandaloneContainerApplication].getName).asJava) - } - - - private class JarBundleImpl(location: URL) extends BundleImpl { - override val getBundleId = nextBundleId.getAndIncrement.asInstanceOf[Long] - - private val headers = retrieveHeaders(location) - - override def getHeaders: Dictionary[String, String] = headers - override val getSymbolicName = headers.get("Bundle-SymbolicName") - override val getVersion = Version.parseVersion(headers.get("Bundle-Version")) - - - private def retrieveHeaders(location: URL) = { - val jarFile = new JarFile(location.getFile) - try { - val attributes = jarFile.getManifest.getMainAttributes - new Hashtable[String, String](attributes.entrySet().asScala.map( entry => entry.getKey.toString -> entry.getValue.toString).toMap.asJava) - } finally { - jarFile.close() - } - } - } - - private object BundleContextImpl extends BundleContext { - private val bundleImpl = SystemBundleImpl - - override def getProperty(key: String) = null - override def getBundle = bundleImpl - override def installBundle(location: String, input: InputStream) = throw new UnsupportedOperationException - override def installBundle(location: String) = throw new UnsupportedOperationException - - override def getBundle(id: Long) = bundleImpl - override def getBundles = Array(bundleImpl) - override def getBundle(location: String) = bundleImpl - - override def addServiceListener(listener: ServiceListener, filter: String) {} - override def addServiceListener(listener: ServiceListener) {} - override def removeServiceListener(listener: ServiceListener) {} - override def addBundleListener(listener: BundleListener) {} - override def removeBundleListener(listener: BundleListener) {} - - override def addFrameworkListener(listener: FrameworkListener) {} - override def removeFrameworkListener(listener: FrameworkListener) {} - - override def registerService(clazzes: Array[String], service: Any, properties: Dictionary[String, _]) = throw new UnsupportedOperationException - override def registerService(clazz: String, service: Any, properties: Dictionary[String, _]) = null - override def registerService[S](clazz: Class[S], service: S, properties: Dictionary[String, _]) = throw new UnsupportedOperationException - override def getServiceReferences(clazz: String, filter: String) = throw new UnsupportedOperationException - override def getAllServiceReferences(clazz: String, filter: String) = throw new UnsupportedOperationException - override def getServiceReference(clazz: String) = throw new UnsupportedOperationException - override def getServiceReference[S](clazz: Class[S]) = throw new UnsupportedOperationException - override def getServiceReferences[S](clazz: Class[S], filter: String) = Collections.emptyList() - override def getService[S](reference: ServiceReference[S]) = throw new UnsupportedOperationException - override def ungetService(reference: ServiceReference[_]) = throw new UnsupportedOperationException - override def getDataFile(filename: String) = throw new UnsupportedOperationException - override def createFilter(filter: String) = throw new UnsupportedOperationException - - override def registerService[S](aClass: Class[S], serviceFactory: ServiceFactory[S], dictionary: Dictionary[String, _]): ServiceRegistration[S] = throw new UnsupportedOperationException - override def getServiceObjects[S](serviceReference: ServiceReference[S]): ServiceObjects[S] = throw new UnsupportedOperationException - } - -} diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala deleted file mode 100644 index 91634250fc5..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.impl - -import java.nio.file.Files -import com.yahoo.text.Utf8 - -/** - * @author tonytv - */ -final class StandaloneContainerRunner { - - - -} - -object StandaloneContainerRunner { - def createApplicationPackage(servicesXml: String) = { - val applicationDir = Files.createTempDirectory("application") - - val servicesXmlFile = applicationDir.resolve("services.xml"); - var content = servicesXml; - if ( ! servicesXml.startsWith("<?xml")) - content = """<?xml version="1.0" encoding="utf-8" ?>""" + '\n' + servicesXml - Files.write(servicesXmlFile, Utf8.toBytes(content)) - applicationDir - } -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala deleted file mode 100644 index 61128347319..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -/** - * @author tonytv - */ -trait Converter[T] { - def convert(s: String): T -} - -object Converter { - def toConverter[T](f: String => T) = new Converter[T] { - override def convert(s: String) = f(s) - } - - implicit val intConverter = toConverter(_.toInt) - implicit val longConverter = toConverter(_.toLong) - implicit val boolConverter = toConverter(_.toBoolean) - implicit val stringConverter = toConverter(identity) - - implicit val javaIntegerConverter:Converter[Integer] = toConverter(_.toInt) - implicit val javaLongConverter:Converter[java.lang.Long] = toConverter(_.toLong) - implicit val javaBooleanConverter:Converter[java.lang.Boolean] = toConverter(_.toBoolean) - - -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala deleted file mode 100644 index 2aab88d8319..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -/** - * @author tonytv - * TODO: copied from standalone-container. Move to separate lib module instead. - */ -object Environment { - def optionalInstallVariable(name: String) = { - env(name.replace(".", "__")). - orElse(systemProperty(name)) //for unit testing - } - - def installVariable(name: String) = { - optionalInstallVariable(name). - getOrElse { - throw new IllegalStateException("Environment variable not set: " + name) - } - } - - def env(name: String) = Option(System.getenv(name)) - def systemProperty(name: String) = Option(System.getProperty(name)) -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala deleted file mode 100644 index 6507b4c72f0..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import java.io.File -import java.lang.reflect.Constructor -import java.nio.file.Path -import java.util -import java.util.concurrent.TimeUnit - -import com.yahoo.config.FileReference -import com.yahoo.config.application.api.FileRegistry -import com.yahoo.config.application.api.FileRegistry.Entry -import com.yahoo.container.standalone.LocalFileDb._ -import com.yahoo.filedistribution.fileacquirer.FileAcquirer -import com.yahoo.net.HostName - -import scala.collection.JavaConverters._ -import scala.collection.mutable - - -/** - * FileAcquirer and FileRegistry working on a local directory. - * @author tonytv - */ -class LocalFileDb(appPath: Path) extends FileAcquirer with FileRegistry { - private val fileReferenceToFile = mutable.Map[FileReference, File]() - - /** *** FileAcquirer overrides *****/ - def waitFor(reference: FileReference, l: Long, timeUnit: TimeUnit): File = { - synchronized { - fileReferenceToFile.get(reference).getOrElse { - throw new RuntimeException("Invalid file reference " + reference) - } - } - } - - override def shutdown() {} - - /** *** FileRegistry overrides *****/ - def addFile(relativePath: String): FileReference = { - val file = appPath.resolve(relativePath).toFile - if (!file.exists) { - throw new RuntimeException("The file does not exist: " + file.getPath) - } - - val fileReference: FileReference = fileReferenceConstructor.newInstance("LocalFileDb:" + relativePath) - fileReferenceToFile.put(fileReference, file) - fileReference - } - - def fileSourceHost: String = - HostName.getLocalhost - - def allRelativePaths: java.util.Set[String] = { - new java.util.HashSet(fileReferenceToFile.values.map(_.getPath).asJavaCollection) - } - - override def export(): util.List[Entry] = { - new java.util.ArrayList(fileReferenceToFile.keys.map{ (ref: FileReference) => new Entry(fileReferenceToFile.get(ref).get.getPath, ref)}.asJavaCollection) - } - - override def addUri(uri: String): FileReference = { - throw new RuntimeException("addUri(uri: String) is not implemented here."); - } -} - -object LocalFileDb { - private def createFileReferenceConstructor: Constructor[FileReference] = { - val method: Constructor[FileReference] = classOf[FileReference].getDeclaredConstructor(classOf[String]) - method.setAccessible(true) - method - } - - private val fileReferenceConstructor: Constructor[FileReference] = createFileReferenceConstructor -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala deleted file mode 100644 index 5271cd400d4..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import java.io.{File, IOException} -import java.lang.{Boolean => JBoolean} -import java.nio.file.{FileSystems, Files, Path, Paths} - -import com.google.inject.name.Names -import com.google.inject.{AbstractModule, Inject, Injector, Key} -import com.yahoo.collections.CollectionUtil.first -import com.yahoo.config.application.api.{ApplicationPackage, FileRegistry} -import com.yahoo.config.model.application.provider._ -import com.yahoo.config.model.builder.xml.XmlHelper -import com.yahoo.config.model.deploy.DeployState -import com.yahoo.config.model.{ApplicationConfigProducerRoot, ConfigModelRepo} -import com.yahoo.config.provision.Zone -import com.yahoo.container.di.config.SubscriberFactory -import com.yahoo.container.jdisc.ConfiguredApplication -import com.yahoo.container.standalone.Environment._ -import com.yahoo.container.standalone.StandaloneContainerApplication._ -import com.yahoo.io.IOUtils -import com.yahoo.jdisc.application.Application -import com.yahoo.text.XML -import com.yahoo.vespa.defaults.Defaults -import com.yahoo.vespa.model.VespaModel -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder -import com.yahoo.vespa.model.container.{Container, ContainerModel} -import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking -import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ContainerModelBuilder} -import org.w3c.dom.Element - -import scala.collection.JavaConverters._ -import scala.util.Try - -/** - * @author Tony Vaagenes - * @author gjoranv - */ -class StandaloneContainerApplication @Inject()(injector: Injector) extends Application { - - ConfiguredApplication.ensureVespaLoggingInitialized() - - val applicationPath: Path = injectedApplicationPath.getOrElse(installApplicationPath) - - val distributedFiles = new LocalFileDb(applicationPath) - - val configModelRepo = Try { injector.getInstance(Key.get(classOf[ConfigModelRepo], configModelRepoName))}.getOrElse(new ConfigModelRepo) - - val networkingOption = Try { - injector.getInstance(Key.get(classOf[JBoolean], Names.named(disableNetworkingAnnotation))) - }.map { - case JBoolean.TRUE => Networking.disable - case JBoolean.FALSE => Networking.enable - }.getOrElse(Networking.enable) - - val (modelRoot, container) = withTempDir( - preprocessedApplicationDir => createContainerModel(applicationPath, distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo)) - - val configuredApplication = createConfiguredApplication(container) - - def createConfiguredApplication(container: Container): Application = { - val augmentedInjector = injector.createChildInjector(new AbstractModule { - def configure() { - bind(classOf[SubscriberFactory]).toInstance(new StandaloneSubscriberFactory(modelRoot)) - } - }) - - System.setProperty("config.id", container.getConfigId) //TODO: DRY - augmentedInjector.getInstance(classOf[ConfiguredApplication]) - } - - def injectedApplicationPath = Try { - injector.getInstance(Key.get(classOf[Path], applicationPathName)) - }.toOption - - def installApplicationPath = path(installVariable(applicationLocationInstallVariable)) - - override def start() { - try { - com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles) - configuredApplication.start() - } - catch { - case e: Exception => { com.yahoo.container.Container.resetInstance(); throw e; } - } - } - - override def stop() { - configuredApplication.stop() - } - - override def destroy() { - com.yahoo.container.Container.resetInstance() - configuredApplication.destroy() - } -} - -object StandaloneContainerApplication { - val packageName = "standalone_jdisc_container" - val applicationLocationInstallVariable = s"$packageName.app_location" - val deploymentProfileInstallVariable = s"$packageName.deployment_profile" - - val applicationPathName = Names.named(applicationLocationInstallVariable) - - val disableNetworkingAnnotation = "JDisc.disableNetworking" - val configModelRepoName = Names.named("ConfigModelRepo") - val configDefinitionRepo = new StaticConfigDefinitionRepo() - - val defaultTmpBaseDir = Defaults.getDefaults().underVespaHome("tmp") - val tmpDirName = "standalone_container" - - private def withTempDir[T](f: File => T): T = { - val tmpDir = createTempDir() - try { - f(tmpDir) - } finally { - IOUtils.recursiveDeleteDir(tmpDir) - } - } - - private def createTempDir(): File = { - def getBaseDir: Path = { - val tmpBaseDir = - if (new File(defaultTmpBaseDir).exists()) - defaultTmpBaseDir - else - System.getProperty("java.io.tmpdir") - - Paths.get(tmpBaseDir) - } - - val basePath: Path = getBaseDir - val tmpDir = Files.createTempDirectory(basePath, tmpDirName) - tmpDir.toFile - } - - private def validateApplication(applicationPackage: ApplicationPackage) = { - try { - applicationPackage.validateXML() - } catch { - case e: IOException => throw new IllegalArgumentException(e) - } - } - - def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = { - optionalInstallVariable(deploymentProfileInstallVariable) match { - case None => new ContainerModelBuilder(true, networkingOption) - case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables) - case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'") - } - } - - def createContainerModel(applicationPath: Path, - fileRegistry: FileRegistry, - preprocessedApplicationDir: File, - networkingOption: Networking, - configModelRepo: ConfigModelRepo = new ConfigModelRepo): (VespaModel, Container) = { - val logger = new BaseDeployLogger - val rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile).includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build() - val applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), logger) - validateApplication(applicationPackage) - val deployState = new DeployState.Builder(). - applicationPackage(applicationPackage). - fileRegistry(fileRegistry). - deployLogger(logger). - configDefinitionRepo(configDefinitionRepo). - build(true) - - val root = VespaModel.createIncomplete(deployState) - val vespaRoot = new ApplicationConfigProducerRoot(root, - "vespa", - deployState.getDocumentModel, - deployState.getProperties.vespaVersion(), - deployState.getProperties.applicationId()) - - val spec = containerRootElement(applicationPackage) - val containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec) - containerModel.getCluster().prepare() - DeprecationSuppressor.initializeContainerModel(containerModel, configModelRepo) - val container = first(containerModel.getCluster().getContainers) - - // TODO: Separate out model finalization from the VespaModel constructor, - // such that the above and below code to finalize the container can be - // replaced by root.finalize(); - - initializeContainer(container, spec) - - root.freezeModelTopology() - (root, container) - } - - def initializeContainer(container: Container, spec: Element) { - val host = container.getRoot.getHostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC) - - container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec)) - container.setHostResource(host) - container.initService() - } - - def getJDiscInServices(element: Element): Element = { - def nameAndId(elements: List[Element]): List[String] = { - elements map { e => s"${e.getNodeName} id='${e.getAttribute("id")}'" } - } - - val jDiscElements = ContainerModelBuilder.configModelIds.asScala flatMap { name => XML.getChildren(element, name.getName).asScala } - jDiscElements.toList match { - case List(e) => e - case Nil => throw new RuntimeException("No jdisc element found under services.") - case multipleElements: List[Element] => throw new RuntimeException("Found multiple JDisc elements: " + nameAndId(multipleElements).mkString(", ")) - } - } - - def containerRootElement(applicationPackage: ApplicationPackage) : Element = { - val element = XmlHelper.getDocument(applicationPackage.getServices).getDocumentElement - val nodeName = element.getNodeName - - if (ContainerModelBuilder.configModelIds.asScala.map(_.getName).contains(nodeName)) element - else getJDiscInServices(element) - } - - def path(s: String) = FileSystems.getDefault.getPath(s) - - // Ugly hack required since Scala cannot suppress warnings. https://issues.scala-lang.org/browse/SI-7934 - private object DeprecationSuppressor extends DeprecationSuppressor - @deprecated("", "") - private class DeprecationSuppressor { - def initializeContainerModel(containerModel: ContainerModel, configModelRepo: ConfigModelRepo): Unit = { - containerModel.initialize(configModelRepo) - } - } -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala deleted file mode 100644 index 1bc95f52313..00000000000 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import com.yahoo.config.{ConfigBuilder, ConfigInstance} -import com.yahoo.container.di.ConfigKeyT -import com.yahoo.container.di.config.{Subscriber, SubscriberFactory} -import com.yahoo.container.standalone.StandaloneSubscriberFactory._ -import com.yahoo.vespa.config.ConfigKey -import com.yahoo.vespa.model.VespaModel - -import scala.collection.JavaConverters._ - -/** - * @author tonytv - * @author gjoranv - */ -class StandaloneSubscriberFactory(root: VespaModel) extends SubscriberFactory { - class StandaloneSubscriber(configKeys: Set[ConfigKeyT]) extends Subscriber { - override def configChanged = - generation == 0 - - override def close() {} - - override def internalRedeploy() = { false } - - override def config = { - - def getConfig(key: ConfigKeyT) = { - val builderWithModelConfig = root.getConfig(newBuilderInstance(key), key.getConfigId) - - require(builderWithModelConfig != null, "Invalid config id " + key.getConfigId ) - (key.asInstanceOf[ConfigKey[ConfigInstance]], newConfigInstance(builderWithModelConfig)) - } - - (configKeys map getConfig).toMap.asJava - } - - override def waitNextGeneration() = { - generation += 1 - - if (generation != 0) { - while (!Thread.interrupted()) - Thread.sleep(10000) - } - - generation - } - - //if waitNextGeneration has not yet been called, -1 should be returned - var generation = -1L - - } - - override def getSubscriber(configKeys: java.util.Set[_ <: ConfigKey[_]]) = - new StandaloneSubscriber(configKeys.asScala.toSet.asInstanceOf[Set[ConfigKeyT]]) - - def reloadActiveSubscribers(generation: Long) { - throw new RuntimeException("unsupported") - } -} - -object StandaloneSubscriberFactory { - - private def newBuilderInstance(key: ConfigKeyT) = - builderClass(key).getDeclaredConstructor().newInstance() - - private def builderClass(key: ConfigKeyT) = { - val nestedClasses = key.getConfigClass.getClasses - nestedClasses. - filter {_.getName.equals(key.getConfigClass.getName + "$Builder")}. - head. - asInstanceOf[Class[ConfigInstance.Builder]] - } - - private def newConfigInstance(builder: ConfigBuilder) = - configClass(builder).getConstructor(builder.getClass).newInstance(builder) - - private def configClass(builder: ConfigBuilder) = - builder.getClass.getEnclosingClass.asInstanceOf[Class[ConfigInstance]] - -} diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java new file mode 100644 index 00000000000..1cd110d8106 --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java @@ -0,0 +1,67 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigModelsPluginDir; +import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigServer; +import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigServers; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertThat; + +/** + * @author Ulf Lilleengen + * @author Tony Vaagenes + */ +public class CloudConfigInstallVariablesTest { + + @Test + public void test_configserver_parsing() { + CloudConfigOptions.ConfigServer[] parsed = toConfigServers("myhost.mydomain.com"); + assertThat(parsed.length, is(1)); + } + + @Test + public void port_can_be_configured() { + CloudConfigOptions.ConfigServer[] parsed = toConfigServers("myhost:123"); + int port = parsed[0].port.get(); + assertThat(port, is(123)); + } + + @Test + public void multiple_spaces_are_supported() { + CloudConfigOptions.ConfigServer[] parsed = toConfigServers("test1 test2"); + assertThat(parsed.length, is(2)); + + List<String> hostNames = Arrays.stream(parsed).map(cs -> cs.hostName).collect(Collectors.toList()); + assertThat(hostNames, contains("test1", "test2")); + } + + @Test(expected = IllegalArgumentException.class) + public void missing_port_gives_exception() { + toConfigServer("myhost:"); + } + + @Test(expected = IllegalArgumentException.class) + public void non_numeric_port_gives_exception() { + toConfigServer("myhost:non-numeric"); + } + + @Test + public void string_arrays_are_split_on_spaces() { + String[] parsed = toConfigModelsPluginDir("/home/vespa/foo /home/vespa/bar "); + assertThat(parsed.length, is(2)); + } + + @Test + public void string_arrays_are_split_on_comma() { + String[] parsed = toConfigModelsPluginDir("/home/vespa/foo,/home/vespa/bar,"); + assertThat(parsed.length, is(2)); + } +} diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java new file mode 100644 index 00000000000..a00ffd8b985 --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java @@ -0,0 +1,61 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.collections.Pair; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; + +/** + * Creates a local application from vespa-services fragments. + * + * @author Tony Vaagenes + * @author ollivir + */ +public class StandaloneContainer { + public static String firstContainerId(AbstractConfigProducerRoot root) { + return root.getConfigProducer("container").get().getConfigId(); + } + + interface ThrowingFunction<T, U> { + U apply(T input) throws Exception; + } + + static <T> T withContainerModel(String servicesXml, ThrowingFunction<VespaModel, T> f) throws Exception { + return withTempDirectory(applicationPath -> { + writeServicesXml(applicationPath, servicesXml); + + LocalFileDb distributedFiles = new LocalFileDb(applicationPath); + VespaModel root; + Pair<VespaModel, com.yahoo.vespa.model.container.Container> rc = StandaloneContainerApplication.createContainerModel( + applicationPath, distributedFiles, applicationPath.resolve("preprocesedApp").toFile(), Networking.enable, + new ConfigModelRepo()); + root = rc.getFirst(); + return f.apply(root); + }); + } + + private static <T> T withTempDirectory(ThrowingFunction<Path, T> f) throws Exception { + Path directory = Files.createTempDirectory("application"); + try { + return f.apply(directory); + } finally { + IOUtils.recursiveDeleteDir(directory.toFile()); + } + } + + private static void writeServicesXml(Path applicationPath, String servicesXml) throws IOException { + Path path = applicationPath.resolve("services.xml"); + List<String> output = Arrays.asList("<?xml version=\"1.0\" encoding=\"utf-8\"?>", servicesXml); + Files.write(path, output, StandardCharsets.UTF_8); + } +} diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java index 8d413ade0f0..71668b595a0 100644 --- a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.standalone; import com.google.inject.Module; @@ -25,7 +25,6 @@ import static org.junit.Assert.assertThat; /** * @author Einar M R Rosenvinge - * @since 5.22.0 */ public class StandaloneContainerActivatorTest { @@ -100,7 +99,7 @@ public class StandaloneContainerActivatorTest { private static Module newAppDirBinding(final Path applicationDir) { return binder -> binder.bind(Path.class) - .annotatedWith(StandaloneContainerApplication.applicationPathName()) + .annotatedWith(StandaloneContainerApplication.APPLICATION_PATH_NAME) .toInstance(applicationDir); } diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java new file mode 100644 index 00000000000..6d4abc84dbc --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerTest.java @@ -0,0 +1,74 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.vespa.model.AbstractService; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author Tony Vaagenes + * @author gjoranv + * @author ollivir + */ + +public class StandaloneContainerTest { + private static final String PLAIN_XML = "<container version=\"1.0\" />"; + + @Test + public void container_is_allowed_root_element() throws Exception { + StandaloneContainer.withContainerModel(PLAIN_XML, root -> null); + } + + @Test + public void services_is_allowed_root_element() throws Exception { + String servicesXml = "<services>" + // + "<container version=\"1.0\" />" + // + "</services>"; + + StandaloneContainer.withContainerModel(servicesXml, root -> null); + } + + @Test(expected = Exception.class) + public void multiple_container_elements_cannot_be_deployed() throws Exception { + String twoContainersXml = "<services>" + // + "<container id=\"container-1\" version=\"1.0\" />" + // + "<container id=\"container-2\" version=\"1.0\" />" + // + "</services>"; + + StandaloneContainer.withContainerModel(twoContainersXml, root -> null); + } + + @Test + public void application_preprocessor_is_run() throws Exception { + String servicesXml = "<services xmlns:preprocess=\"properties\">" + // + "<preprocess:properties>" + // + "<container_id>container-1</container_id>" + // + "</preprocess:properties>" + // + "<container id=\"${container_id}\" version=\"1.0\" />" + // + "</services>"; + + StandaloneContainer.withContainerModel(servicesXml, root -> { + assertTrue(root.getConfigProducer("container-1/standalone").isPresent()); + return null; + }); + } + + @Test + public void no_default_ports_are_enabled_when_using_http() throws Exception { + String xml = "<jdisc version=\"1.0\">" + // + "<http>" + // + "<server port=\"4000\" id=\"server1\" />" + // + "</http>" + // + "</jdisc>"; + + StandaloneContainer.withContainerModel(xml, root -> { + AbstractService container = (AbstractService) root.getConfigProducer("jdisc/standalone").get(); + System.out.println("portCnt: " + container.getPortCount()); + System.out.println("numPorts: " + container.getNumPortsAllocated()); + assertEquals(1, container.getNumPortsAllocated()); + return null; + }); + } +} diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java new file mode 100644 index 00000000000..dd755e8e6dd --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.container.BundlesConfig; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.di.config.Subscriber; +import com.yahoo.vespa.config.ConfigKey; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static com.yahoo.container.standalone.StandaloneContainer.withContainerModel; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.number.OrderingComparison.greaterThan; +import static org.junit.Assert.assertThat; + +/** + * @author Tony Vaagenes + * @author ollivir + */ +public class StandaloneSubscriberTest { + private static ConfigKey<ConfigInstance> bundlesKey = key("bundles"); + private static ConfigKey<ConfigInstance> componentsKey = key("components"); + + private static ConfigKey<ConfigInstance> key(String name) { + return new ConfigKey<>(name, "container", "container"); + } + + @Test + @Ignore + public void standalone_subscriber() throws Exception { + withContainerModel("<container version=\"1.0\"></container>", root -> { + Set<ConfigKey<ConfigInstance>> keys = new HashSet<>(); + keys.add(bundlesKey); + keys.add(componentsKey); + Subscriber subscriber = new StandaloneSubscriberFactory(root).getSubscriber(keys); + Map<ConfigKey<ConfigInstance>, ConfigInstance> config = subscriber.config(); + assertThat(config.size(), is(2)); + + BundlesConfig bundlesConfig = (BundlesConfig) config.get(bundlesKey); + ComponentsConfig componentsConfig = (ComponentsConfig) config.get(componentsKey); + + assertThat(bundlesConfig.bundle().size(), is(0)); + assertThat(componentsConfig.components().size(), greaterThan(10)); + return null; + }); + } +} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala deleted file mode 100644 index d4baea43ba2..00000000000 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.scala +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import com.yahoo.container.standalone.CloudConfigInstallVariables.{toConfigModelsPluginDir, toConfigServer, toConfigServers} -import org.hamcrest.CoreMatchers.is -import org.hamcrest.Matchers.arrayContaining -import org.junit.Assert.assertThat -import org.junit.Test - -/** - * @author Ulf Lilleengen - * @author Tony Vaagenes - */ -class CloudConfigInstallVariablesTest { - - @Test - def test_configserver_parsing { - val parsed = toConfigServers("myhost.mydomain.com") - assertThat(parsed.length, is(1)) - } - - @Test - def port_can_be_configured { - val parsed = toConfigServers("myhost:123") - val port: Int = parsed(0).port.get() - assertThat(port, is(123)) - } - - @Test - def multiple_spaces_are_supported { - val parsed = toConfigServers("test1 test2") - assertThat(parsed.size, is(2)) - - val hostNames = parsed.map(_.hostName) - assertThat(hostNames, arrayContaining("test1", "test2")) - } - - @Test(expected = classOf[IllegalArgumentException]) - def missing_port_gives_exception { - toConfigServer("myhost:") - } - - @Test(expected = classOf[IllegalArgumentException]) - def non_numeric_port_gives_exception { - toConfigServer("myhost:non-numeric") - } - - @Test - def string_arrays_are_split_on_spaces { - val parsed = toConfigModelsPluginDir("/home/vespa/foo /home/vespa/bar ") - assertThat(parsed.size, is(2)) - } - - @Test - def string_arrays_are_split_on_comma { - val parsed = toConfigModelsPluginDir("/home/vespa/foo,/home/vespa/bar,") - assertThat(parsed.size, is(2)) - } -} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala deleted file mode 100644 index 33f9a2e8594..00000000000 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import com.yahoo.config.model.producer.AbstractConfigProducerRoot -import com.yahoo.config.model.test.MockRoot -import com.yahoo.container.Container -import com.yahoo.jdisc.test.TestDriver -import scala.xml.Node -import com.yahoo.vespa.model.VespaModel -import com.yahoo.io.IOUtils -import java.nio.file.{Files, Path} -import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking - -/** - * Creates a local application from vespa-services fragments. - * - * @author tonytv - */ -object StandaloneContainer { - def firstContainerId(root: AbstractConfigProducerRoot): String = { - root.getConfigProducer("container").get().getConfigId - } - - def withStandaloneContainer[T](containerNode: Node) { - withTempDirectory { applicationDirectory => - System.setProperty(StandaloneContainerApplication.applicationLocationInstallVariable, applicationDirectory.toString) - createServicesXml(applicationDirectory, containerNode) - - val driver = TestDriver.newInjectedApplicationInstance(classOf[StandaloneContainerApplication]) - driver.close() - Container.resetInstance() - } - } - - def withContainerModel[T](containerNode: Node)(f: VespaModel => T) { - withTempDirectory { applicationPath => - createServicesXml(applicationPath, containerNode) - - val distributedFiles = new LocalFileDb(applicationPath) - val (root, container) = StandaloneContainerApplication.createContainerModel( - applicationPath, - distributedFiles, - applicationPath.resolve("preprocesedApp").toFile, - networkingOption = Networking.enable) - f(root) - } - } - - private def withTempDirectory[T](f : Path => T) : T = { - val directory = Files.createTempDirectory("application") - try { - f(directory) - } finally { - IOUtils.recursiveDeleteDir(directory.toFile) - } - } - - private def createServicesXml(applicationPath : Path, - containerNode: Node) { - - scala.xml.XML.save(applicationPath.resolve("services.xml").toFile.getAbsolutePath, - containerNode, xmlDecl = true) - } -} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala deleted file mode 100644 index 87bef2efd95..00000000000 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - - -import com.yahoo.container.standalone.StandaloneContainerTest._ -import com.yahoo.vespa.model.AbstractService -import org.junit.Assert._ -import org.junit.Test - -import scala.util.Try - - -/** - * @author tonytv - * @author gjoranv - */ - -class StandaloneContainerTest { - @Test - def container_is_allowed_root_element() { - StandaloneContainer.withContainerModel(plainXml) { root => } - } - - @Test - def services_is_allowed_root_element() { - val servicesXml = - <services> - <container version="1.0" /> - </services> - - StandaloneContainer.withContainerModel(servicesXml) { root => } - } - - @Test - def multiple_container_elements_cannot_be_deployed() { - val twoContainersXml = - <services> - <container id="container-1" version="1.0" /> - <container id="container-2" version="1.0" /> - </services> - - assertTrue( - Try { - StandaloneContainer.withContainerModel(twoContainersXml) { root => } - }.isFailure) - } - - @Test - def application_preprocessor_is_run() { - val servicesXml = - <services xmlns:preprocess="properties"> - <preprocess:properties> - <container_id>container-1</container_id> - </preprocess:properties> - <container id="${container_id}" version="1.0" /> - </services> - StandaloneContainer.withContainerModel(servicesXml) { - root => - assertTrue(root.getConfigProducer("container-1/standalone").isPresent) - } - } - - @Test - def no_default_ports_are_enabled_when_using_http() { - val xml = - <jdisc version="1.0"> - <http> - <server port="4000" id="server1" /> - </http> - </jdisc> - - StandaloneContainer.withContainerModel(xml) { root => - val container = root.getConfigProducer("jdisc/standalone").get().asInstanceOf[AbstractService] - println("portCnt: " + container.getPortCount) - println("numPorts: " + container.getNumPortsAllocated) - assertEquals(1, container.getNumPortsAllocated) - } - } - -} - -object StandaloneContainerTest { - - val plainXml = <container version="1.0" /> -} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala deleted file mode 100644 index ab6f486c748..00000000000 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.standalone - -import org.junit.{Ignore, Test} -import org.junit.Assert.assertThat -import org.hamcrest.CoreMatchers.is -import org.hamcrest.number.OrderingComparison.greaterThan - -import StandaloneContainer.withContainerModel -import com.yahoo.vespa.config.ConfigKey -import com.yahoo.config.ConfigInstance -import com.yahoo.container.{ComponentsConfig, BundlesConfig, di} -import scala.collection.JavaConverters._ - -/** - * @author tonytv - */ -class StandaloneSubscriberTest { - val bundlesKey = key("bundles") - val componentsKey = key("components") - - def key(name: String) = new ConfigKey(name, "container", "container").asInstanceOf[ConfigKey[ConfigInstance]] - - def box(i: Int) = java.lang.Integer.valueOf(i) - - @Test - @Ignore - def standalone_subscriber() { - withContainerModel(<container version="1.0"> </container>) { root => - val subscriber = new StandaloneSubscriberFactory(root).getSubscriber(Set(bundlesKey, componentsKey).asJava) - val config = subscriber.config.asScala - assertThat(config.size, is(2)) - - val bundlesConfig = config(bundlesKey).asInstanceOf[BundlesConfig] - val componentsConfig = config(componentsKey).asInstanceOf[ComponentsConfig] - - assertThat(bundlesConfig.bundle().size(), is(0)) - assertThat(box(componentsConfig.components().size()), greaterThan(box(10))) - } - } -} diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index a271fbccf50..3b72585b076 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -14,6 +14,8 @@ #include <vespa/documentapi/messagebus/messages/getdocumentmessage.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> using document::test::makeDocumentBucket; @@ -27,6 +29,7 @@ struct CommunicationManagerTest : public CppUnit::TestFixture { void testRepliesAreDequeuedInFifoOrder(); void bucket_space_config_can_be_updated_live(); void unmapped_bucket_space_documentapi_request_returns_error_reply(); + void unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply(); static constexpr uint32_t MESSAGE_WAIT_TIME_SEC = 60; @@ -51,6 +54,7 @@ struct CommunicationManagerTest : public CppUnit::TestFixture { CPPUNIT_TEST(testRepliesAreDequeuedInFifoOrder); CPPUNIT_TEST(bucket_space_config_can_be_updated_live); CPPUNIT_TEST(unmapped_bucket_space_documentapi_request_returns_error_reply); + CPPUNIT_TEST(unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply); CPPUNIT_TEST_SUITE_END(); }; @@ -267,13 +271,21 @@ struct CommunicationManagerFixture { } ~CommunicationManagerFixture(); - std::unique_ptr<documentapi::GetDocumentMessage> documentapi_message_for_space(const char* space) { - auto cmd = std::make_unique<documentapi::GetDocumentMessage>( - document::DocumentId(vespalib::make_string("id::%s::stuff", space))); + template <typename T> + std::unique_ptr<T> documentapi_message_for_space(const char *space) { + auto cmd = std::make_unique<T>(document::DocumentId(vespalib::make_string("id::%s::stuff", space))); // Bind reply handling to our own mock handler cmd->pushHandler(reply_handler); return cmd; } + + std::unique_ptr<documentapi::RemoveDocumentMessage> documentapi_remove_message_for_space(const char *space) { + return documentapi_message_for_space<documentapi::RemoveDocumentMessage>(space); + } + + std::unique_ptr<documentapi::GetDocumentMessage> documentapi_get_message_for_space(const char *space) { + return documentapi_message_for_space<documentapi::GetDocumentMessage>(space); + } }; CommunicationManagerFixture::~CommunicationManagerFixture() = default; @@ -298,8 +310,8 @@ void CommunicationManagerTest::bucket_space_config_can_be_updated_live() { config.documenttype.emplace_back(doc_type("bar", "global")); f.comm_mgr->updateBucketSpacesConfig(config); - f.comm_mgr->handleMessage(f.documentapi_message_for_space("bar")); - f.comm_mgr->handleMessage(f.documentapi_message_for_space("foo")); + f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("bar")); + f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("foo")); f.bottom_link->waitForMessages(2, MESSAGE_WAIT_TIME_SEC); auto cmd1 = f.bottom_link->getCommand(0); @@ -310,7 +322,7 @@ void CommunicationManagerTest::bucket_space_config_can_be_updated_live() { config.documenttype[1] = doc_type("bar", "default"); f.comm_mgr->updateBucketSpacesConfig(config); - f.comm_mgr->handleMessage(f.documentapi_message_for_space("bar")); + f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("bar")); f.bottom_link->waitForMessages(3, MESSAGE_WAIT_TIME_SEC); auto cmd3 = f.bottom_link->getCommand(2); @@ -328,7 +340,7 @@ void CommunicationManagerTest::unmapped_bucket_space_documentapi_request_returns CPPUNIT_ASSERT_EQUAL(uint64_t(0), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); - f.comm_mgr->handleMessage(f.documentapi_message_for_space("fluff")); + f.comm_mgr->handleMessage(f.documentapi_remove_message_for_space("fluff")); CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size()); auto& reply = *f.reply_handler.replies[0]; CPPUNIT_ASSERT(reply.hasErrors()); @@ -337,4 +349,23 @@ void CommunicationManagerTest::unmapped_bucket_space_documentapi_request_returns CPPUNIT_ASSERT_EQUAL(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); } +// Legacy DocumentAPI routing protocols will send Gets to _all_ clusters even +// if they do not contain a particular document type. By sending an empty reply +// we signal a mergeable "not found" to the sender rather than a non-mergeable +// fatal error. +void CommunicationManagerTest::unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply() { + CommunicationManagerFixture f; + + BucketspacesConfigBuilder config; + config.documenttype.emplace_back(doc_type("foo", "default")); + f.comm_mgr->updateBucketSpacesConfig(config); + + f.comm_mgr->handleMessage(f.documentapi_get_message_for_space("fluff")); + CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size()); + auto& reply = *f.reply_handler.replies[0]; + CPPUNIT_ASSERT(!reply.hasErrors()); + auto& get_reply = dynamic_cast<documentapi::GetDocumentReply&>(reply); + CPPUNIT_ASSERT(!get_reply.hasDocument()); +} + } // storage diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index a08445ca3d2..471ce7e2b27 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -236,6 +236,7 @@ PendingClusterState::onRequestBucketInfoReply(const std::shared_ptr<api::Request if (result == api::ReturnCode::Result::ENCODE_ERROR) { // Handle failure to encode bucket space due to use of old storage api // protocol. Pretend that request succeeded with no buckets returned. + // TODO remove this workaround for Vespa 7 LOG(debug, "Got ENCODE_ERROR, pretending success with no buckets"); } else if (!result.success()) { framework::MilliSecTime resendTime(_clock); diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 22193d7e246..94a151bcdc1 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -19,6 +19,7 @@ #include <vespa/log/bufferedlogger.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> LOG_SETUP(".communication.manager"); @@ -258,8 +259,17 @@ void CommunicationManager::fail_with_unresolvable_bucket_space( { LOG(debug, "Could not map DocumentAPI message to internal bucket: %s", error_message.c_str()); MBUS_TRACE(msg->getTrace(), 6, "Communication manager: Failing message as its document type has no known bucket space mapping"); - std::unique_ptr<mbus::Reply> reply(new mbus::EmptyReply()); - reply->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_REJECTED, error_message)); + std::unique_ptr<mbus::Reply> reply; + if (msg->getType() == documentapi::DocumentProtocol::MESSAGE_GETDOCUMENT) { + // HACK: to avoid breaking legacy routing of GetDocumentMessages to _all_ clusters + // regardless of them having a document type or not, we remap missing bucket spaces + // to explicit Not Found replies (empty document GetDocumentReply). + // TODO remove this workaround for Vespa 7 + reply = std::make_unique<documentapi::GetDocumentReply>(std::shared_ptr<document::Document>()); + } else { + reply = std::make_unique<mbus::EmptyReply>(); + reply->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_REJECTED, error_message)); + } msg->swapState(*reply); _metrics.bucketSpaceMappingFailures.inc(); _messageBusSession->reply(std::move(reply)); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index ad98d64b173..6bb2ca31ec1 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -206,10 +206,8 @@ StorageNode::initialize() _chain.reset(createChain().release()); - if (_component->enableMultipleBucketSpaces()) { - assert(_communicationManager != nullptr); - _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); - } + assert(_communicationManager != nullptr); + _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); // Start the metric manager, such that it starts generating snapshots // and the like. Note that at this time, all metrics should hopefully @@ -359,9 +357,7 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) if (_newBucketSpacesConfig) { _bucketSpacesConfig = std::move(_newBucketSpacesConfig); _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); - if (_component->enableMultipleBucketSpaces()) { - _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); - } + _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index f1e96cc1631..a1be0def20b 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -60,8 +60,7 @@ ProtocolSerialization5_0::getBucketInfo(document::ByteBuffer& buf) const } void -ProtocolSerialization5_0::putBucketInfo( - const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const +ProtocolSerialization5_0::putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const { buf.putInt(info.getChecksum()); buf.putInt(info.getDocumentCount()); @@ -71,8 +70,7 @@ ProtocolSerialization5_0::putBucketInfo( } void -ProtocolSerialization5_0::onEncodeReply( - GBBuf& buf, const api::StorageReply& msg) const +ProtocolSerialization5_0::onEncodeReply(GBBuf& buf, const api::StorageReply& msg) const { SH::putReturnCode(msg.getResult(), buf); buf.putLong(msg.getMsgId()); @@ -80,8 +78,7 @@ ProtocolSerialization5_0::onEncodeReply( } void -ProtocolSerialization5_0::onDecodeReply(BBuf& buf, - api::StorageReply& msg) const +ProtocolSerialization5_0::onDecodeReply(BBuf& buf, api::StorageReply& msg) const { msg.setResult(SH::getReturnCode(buf)); msg.forceMsgId(SH::getLong(buf)); @@ -89,8 +86,7 @@ ProtocolSerialization5_0::onDecodeReply(BBuf& buf, } void -ProtocolSerialization5_0::onEncodeCommand( - GBBuf& buf, const api::StorageCommand& msg) const +ProtocolSerialization5_0::onEncodeCommand(GBBuf& buf, const api::StorageCommand& msg) const { buf.putLong(msg.getMsgId()); buf.putByte(msg.getPriority()); @@ -99,8 +95,7 @@ ProtocolSerialization5_0::onEncodeCommand( } void -ProtocolSerialization5_0::onDecodeCommand(BBuf& buf, - api::StorageCommand& msg) const +ProtocolSerialization5_0::onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const { msg.forceMsgId(SH::getLong(buf)); uint8_t priority = SH::getByte(buf); @@ -118,15 +113,13 @@ ProtocolSerialization5_0::ProtocolSerialization5_0( { } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::PutReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::PutReply& msg) const { buf.putBoolean(msg.wasFound()); onEncodeBucketInfoReply(buf, msg); } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::PutCommand& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::PutCommand& msg) const { SH::putDocument(msg.getDocument().get(), buf); putBucket(msg.getBucket(), buf); @@ -141,24 +134,22 @@ ProtocolSerialization5_0::onDecodePutCommand(BBuf& buf) const document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); document::Bucket bucket = getBucket(buf); api::Timestamp ts(SH::getLong(buf)); - api::PutCommand::UP msg(new api::PutCommand(bucket, doc, ts)); + auto msg = std::make_unique<api::PutCommand>(bucket, doc, ts); msg->setUpdateTimestamp(SH::getLong(buf)); onDecodeBucketInfoCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } api::StorageReply::UP ProtocolSerialization5_0::onDecodePutReply(const SCmd& cmd, BBuf& buf) const { bool wasFound = SH::getBoolean(buf); - api::PutReply::UP msg(new api::PutReply( - static_cast<const api::PutCommand&>(cmd), wasFound)); + auto msg = std::make_unique<api::PutReply>(static_cast<const api::PutCommand&>(cmd), wasFound); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::UpdateReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::UpdateReply& msg) const { buf.putLong(msg.getOldTimestamp()); onEncodeBucketInfoReply(buf, msg); @@ -168,14 +159,12 @@ api::StorageReply::UP ProtocolSerialization5_0::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const { api::Timestamp oldTimestamp(SH::getLong(buf)); - api::UpdateReply::UP msg(new api::UpdateReply( - static_cast<const api::UpdateCommand&>(cmd), oldTimestamp)); + auto msg = std::make_unique<api::UpdateReply>(static_cast<const api::UpdateCommand&>(cmd), oldTimestamp); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::GetReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetReply& msg) const { SH::putDocument(msg.getDocument().get(), buf); buf.putLong(msg.getLastModifiedTimestamp()); @@ -188,28 +177,17 @@ ProtocolSerialization5_0::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const try { document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); api::Timestamp lastModified(SH::getLong(buf)); - api::GetReply::UP msg( - new api::GetReply( - static_cast<const api::GetCommand&>(cmd), - doc, - lastModified)); + auto msg = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), doc,lastModified); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } catch (std::exception& e) { - api::GetReply::UP msg( - new api::GetReply( - static_cast<const api::GetCommand&>(cmd), - document::Document::SP(), - 0)); - msg->setResult(api::ReturnCode( - api::ReturnCode::UNPARSEABLE, - e.what())); - return api::StorageReply::UP(msg.release()); + auto msg = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), document::Document::SP(),0); + msg->setResult(api::ReturnCode(api::ReturnCode::UNPARSEABLE, e.what())); + return msg; } } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::RemoveReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RemoveReply& msg) const { buf.putLong(msg.getOldTimestamp()); onEncodeBucketInfoReply(buf, msg); @@ -219,14 +197,12 @@ api::StorageReply::UP ProtocolSerialization5_0::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const { api::Timestamp oldTimestamp(SH::getLong(buf)); - api::RemoveReply::UP msg(new api::RemoveReply( - static_cast<const api::RemoveCommand&>(cmd), oldTimestamp)); + auto msg = std::make_unique<api::RemoveReply>(static_cast<const api::RemoveCommand&>(cmd), oldTimestamp); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::UpdateCommand& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::UpdateCommand& msg) const { document::DocumentUpdate* update = msg.getUpdate().get(); if (update) { @@ -253,24 +229,19 @@ ProtocolSerialization5_0::onDecodeUpdateCommand(BBuf& buf) const if (size != 0) { document::ByteBuffer bbuf(buf.getBufferAtPos(), size); buf.incPos(size); - update.reset(new document::DocumentUpdate(getTypeRepo(), bbuf, - document::DocumentUpdate:: - SerializeVersion:: - SERIALIZE_HEAD)); + update = document::DocumentUpdate::createHEAD(getTypeRepo(), bbuf); } document::Bucket bucket = getBucket(buf); api::Timestamp timestamp(SH::getLong(buf)); - api::UpdateCommand::UP msg( - new api::UpdateCommand(bucket, update, timestamp)); + api::UpdateCommand::UP msg = std::make_unique<api::UpdateCommand>(bucket, update, timestamp); msg->setOldTimestamp(SH::getLong(buf)); onDecodeBucketInfoCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::RevertReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RevertReply& msg) const { onEncodeBucketInfoReply(buf, msg); } @@ -278,31 +249,26 @@ void ProtocolSerialization5_0::onEncode( api::StorageReply::UP ProtocolSerialization5_0::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const { - api::RevertReply::UP msg(new api::RevertReply( - static_cast<const api::RevertCommand&>(cmd))); + auto msg = std::make_unique<api::RevertReply>(static_cast<const api::RevertCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::CreateBucketReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateBucketReply& msg) const { onEncodeBucketInfoReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd, BBuf& buf) const { - api::CreateBucketReply::UP msg(new api::CreateBucketReply( - static_cast<const api::CreateBucketCommand&>(cmd))); + auto msg = std::make_unique<api::CreateBucketReply>(static_cast<const api::CreateBucketCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } void -ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::DeleteBucketCommand& msg) const +ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::DeleteBucketCommand& msg) const { putBucket(msg.getBucket(), buf); onEncodeBucketInfoCommand(buf, msg); @@ -313,32 +279,28 @@ api::StorageCommand::UP ProtocolSerialization5_0::onDecodeDeleteBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::DeleteBucketCommand::UP msg(new api::DeleteBucketCommand(bucket)); + auto msg = std::make_unique<api::DeleteBucketCommand>(bucket); onDecodeBucketInfoCommand(buf, *msg); if (buf.getRemaining() >= SH::BUCKET_INFO_SERIALIZED_SIZE) { msg->setBucketInfo(getBucketInfo(buf)); } - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::DeleteBucketReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::DeleteBucketReply& msg) const { onEncodeBucketInfoReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd, BBuf& buf) const { - api::DeleteBucketReply::UP msg(new api::DeleteBucketReply( - static_cast<const api::DeleteBucketCommand&>(cmd))); + auto msg = std::make_unique<api::DeleteBucketReply>(static_cast<const api::DeleteBucketCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::MergeBucketCommand& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const { ProtocolSerialization4_2::onEncode(buf, msg); @@ -353,8 +315,7 @@ void ProtocolSerialization5_0::onEncode( api::StorageCommand::UP ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const { - api::StorageCommand::UP cmd - = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf); + api::StorageCommand::UP cmd = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf); uint32_t clusterStateVersion = SH::getInt(buf); uint16_t chainSize = SH::getShort(buf); std::vector<uint16_t> chain; @@ -369,24 +330,20 @@ ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const return cmd; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::MergeBucketReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::MergeBucketReply& msg) const { onEncodeBucketReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd, BBuf& buf) const { - api::MergeBucketReply::UP msg(new api::MergeBucketReply( - static_cast<const api::MergeBucketCommand&>(cmd))); + auto msg = std::make_unique<api::MergeBucketReply>(static_cast<const api::MergeBucketCommand&>(cmd)); onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::GetBucketDiffReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetBucketDiffReply& msg) const { const std::vector<api::GetBucketDiffCommand::Entry>& entries(msg.getDiff()); buf.putInt(entries.size()); @@ -397,11 +354,9 @@ void ProtocolSerialization5_0::onEncode( } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd, BBuf& buf) const { - api::GetBucketDiffReply::UP msg(new api::GetBucketDiffReply( - static_cast<const api::GetBucketDiffCommand&>(cmd))); + auto msg = std::make_unique<api::GetBucketDiffReply>(static_cast<const api::GetBucketDiffCommand&>(cmd)); std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -413,24 +368,20 @@ ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd, onDecodeDiffEntry(buf, entries[i]); } onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::ApplyBucketDiffReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::ApplyBucketDiffReply& msg) const { - const std::vector<api::ApplyBucketDiffCommand::Entry>& entries( - msg.getDiff()); + const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg.getDiff()); buf.putInt(entries.size()); for (uint32_t i=0; i<entries.size(); ++i) { onEncodeDiffEntry(buf, entries[i]._entry); buf.putString(entries[i]._docName); buf.putInt(entries[i]._headerBlob.size()); - buf.putBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.putBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); buf.putInt(entries[i]._bodyBlob.size()); - buf.putBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.putBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onEncodeBucketInfoReply(buf, msg); } @@ -439,8 +390,7 @@ api::StorageReply::UP ProtocolSerialization5_0::onDecodeApplyBucketDiffReply(const SCmd& cmd, BBuf& buf) const { - api::ApplyBucketDiffReply::UP msg(new api::ApplyBucketDiffReply( - static_cast<const api::ApplyBucketDiffCommand&>(cmd))); + auto msg = std::make_unique<api::ApplyBucketDiffReply>(static_cast<const api::ApplyBucketDiffCommand&>(cmd)); std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -456,25 +406,21 @@ ProtocolSerialization5_0::onDecodeApplyBucketDiffReply(const SCmd& cmd, buf.incPos(headerSize); } entries[i]._headerBlob.resize(headerSize); - buf.getBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.getBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); uint32_t bodySize = SH::getInt(buf); if (bodySize > buf.getRemaining()) { buf.incPos(bodySize); } entries[i]._bodyBlob.resize(bodySize); - buf.getBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.getBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::SplitBucketReply& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::SplitBucketReply& msg) const { - const std::vector<api::SplitBucketReply::Entry>& entries( - msg.getSplitInfo()); + const std::vector<api::SplitBucketReply::Entry>& entries(msg.getSplitInfo()); buf.putInt(entries.size()); for (std::vector<api::SplitBucketReply::Entry>::const_iterator it = entries.begin(); it != entries.end(); ++it) @@ -486,11 +432,9 @@ void ProtocolSerialization5_0::onEncode( } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, BBuf& buf) const { - api::SplitBucketReply::UP msg(new api::SplitBucketReply( - static_cast<const api::SplitBucketCommand&>(cmd))); + auto msg = std::make_unique<api::SplitBucketReply>(static_cast<const api::SplitBucketCommand&>(cmd)); std::vector<api::SplitBucketReply::Entry>& entries(msg->getSplitInfo()); uint32_t targetCount = SH::getInt(buf); if (targetCount > buf.getRemaining()) { @@ -505,12 +449,11 @@ ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, it->second = getBucketInfo(buf); } onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } void -ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::JoinBucketsCommand& msg) const +ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::JoinBucketsCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putInt(msg.getSourceBuckets().size()); @@ -525,7 +468,7 @@ api::StorageCommand::UP ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::JoinBucketsCommand::UP msg(new api::JoinBucketsCommand(bucket)); + auto msg = std::make_unique<api::JoinBucketsCommand>(bucket); uint32_t size = SH::getInt(buf); if (size > buf.getRemaining()) { // Trigger out of bounds exception rather than out of memory error @@ -537,55 +480,48 @@ ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const } msg->setMinJoinBits(SH::getByte(buf)); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } void -ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::JoinBucketsReply& msg) const +ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::JoinBucketsReply& msg) const { putBucketInfo(msg.getBucketInfo(), buf); onEncodeBucketReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const { - api::JoinBucketsReply::UP msg(new api::JoinBucketsReply( - static_cast<const api::JoinBucketsCommand&>(cmd))); + auto msg = std::make_unique<api::JoinBucketsReply>(static_cast<const api::JoinBucketsCommand&>(cmd)); msg->setBucketInfo(getBucketInfo(buf)); onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } void -ProtocolSerialization5_0::onEncodeBucketInfoReply( - GBBuf& buf, const api::BucketInfoReply& msg) const +ProtocolSerialization5_0::onEncodeBucketInfoReply(GBBuf& buf, const api::BucketInfoReply& msg) const { onEncodeBucketReply(buf, msg); putBucketInfo(msg.getBucketInfo(), buf); } void -ProtocolSerialization5_0::onDecodeBucketInfoReply( - BBuf& buf, api::BucketInfoReply& msg) const +ProtocolSerialization5_0::onDecodeBucketInfoReply(BBuf& buf, api::BucketInfoReply& msg) const { onDecodeBucketReply(buf, msg); msg.setBucketInfo(getBucketInfo(buf)); } void -ProtocolSerialization5_0::onEncodeBucketReply( - GBBuf& buf, const api::BucketReply& msg) const +ProtocolSerialization5_0::onEncodeBucketReply(GBBuf& buf, const api::BucketReply& msg) const { onEncodeReply(buf, msg); buf.putLong(msg.hasBeenRemapped() ? msg.getBucketId().getRawId() : 0); } void -ProtocolSerialization5_0::onDecodeBucketReply( - BBuf& buf, api::BucketReply& msg) const +ProtocolSerialization5_0::onDecodeBucketReply(BBuf& buf, api::BucketReply& msg) const { onDecodeReply(buf, msg); document::BucketId bucket(SH::getLong(buf)); @@ -608,11 +544,9 @@ ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateVisitorReply& ms } api::StorageReply::UP -ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const { - api::CreateVisitorReply::UP msg(new api::CreateVisitorReply( - static_cast<const api::CreateVisitorCommand&>(cmd))); + auto msg = std::make_unique<api::CreateVisitorReply>(static_cast<const api::CreateVisitorCommand&>(cmd)); onDecodeReply(buf, *msg); vdslib::VisitorStatistics vs; @@ -625,11 +559,10 @@ ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, vs.setSecondPassBytesReturned(SH::getLong(buf)); msg->setVisitorStatistics(vs); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_0::onEncode( - GBBuf& buf, const api::RequestBucketInfoCommand& msg) const +void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RequestBucketInfoCommand& msg) const { const std::vector<document::BucketId>& buckets(msg.getBuckets()); buf.putInt(buckets.size()); @@ -663,7 +596,7 @@ ProtocolSerialization5_0::onDecodeRequestBucketInfoCommand(BBuf& buf) const msg.reset(new api::RequestBucketInfoCommand(bucketSpace, distributor, state, SH::getString(buf))); } onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } void diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h index 02c4c0ce0b9..08cec601cce 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -4,13 +4,11 @@ #include <vespa/fastos/types.h> #include <vespa/document/base/globalid.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/growablebytebuffer.h> -namespace storage { -namespace mbusprot { +namespace storage::mbusprot { class SerializationHelper { @@ -60,8 +58,7 @@ public: return api::ReturnCode(result, message); } - static void putReturnCode(const api::ReturnCode& code, - vespalib::GrowableByteBuffer& buf) + static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf) { buf.putInt(code.getResult()); buf.putString(code.getMessage()); @@ -77,18 +74,14 @@ public: return document::GlobalId(&buffer[0]); } - static void putGlobalId(const document::GlobalId& gid, - vespalib::GrowableByteBuffer& buf) + static void putGlobalId(const document::GlobalId& gid, vespalib::GrowableByteBuffer& buf) { buf.putShort(document::GlobalId::LENGTH); for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) { buf.putByte(gid.get()[i]); } } - - static document::Document::UP getDocument( - document::ByteBuffer& buf, - const document::DocumentTypeRepo& repo) + static document::Document::UP getDocument(document::ByteBuffer& buf, const document::DocumentTypeRepo& repo) { uint32_t size = getInt(buf); if (size == 0) { @@ -100,26 +93,7 @@ public: } } - static document::DocumentUpdate::UP getUpdate( - document::ByteBuffer& buf, - const document::DocumentTypeRepo& repo) - { - uint32_t size = getInt(buf); - if (size == 0) { - return document::DocumentUpdate::UP(); - } else { - document::ByteBuffer bbuf(buf.getBufferAtPos(), size); - buf.incPos(size); - return document::DocumentUpdate::UP( - new document::DocumentUpdate(repo, bbuf, - document::DocumentUpdate:: - SerializeVersion:: - SERIALIZE_42)); - } - } - - static void putDocument(document::Document* doc, - vespalib::GrowableByteBuffer& buf) + static void putDocument(document::Document* doc, vespalib::GrowableByteBuffer& buf) { if (doc) { vespalib::nbostream stream; @@ -131,21 +105,6 @@ public: } } - static void putUpdate(document::DocumentUpdate* update, - vespalib::GrowableByteBuffer& buf) - { - if (update) { - vespalib::nbostream stream; - update->serialize42(stream); - buf.putInt(stream.size()); - buf.putBytes(stream.peek(), stream.size()); - } else { - buf.putInt(0); - } - } - }; -} // mbusprot -} // storage - +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 33fca0f161a..f83188f7dd8 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -17,12 +17,12 @@ mbus::string StorageProtocol::NAME = "StorageProtocol"; StorageProtocol::StorageProtocol(const std::shared_ptr<const document::DocumentTypeRepo> repo, const documentapi::LoadTypeSet& loadTypes, - bool activateBucketSpaceSerialization) + bool configForcedBucketSpaceSerialization) : _serializer5_0(repo, loadTypes), _serializer5_1(repo, loadTypes), _serializer5_2(repo, loadTypes), _serializer6_0(repo, loadTypes), - _activateBucketSpaceSerialization(activateBucketSpaceSerialization) + _configForcedBucketSpaceSerialization(configForcedBucketSpaceSerialization) { } @@ -106,7 +106,7 @@ StorageProtocol::encode(const vespalib::Version& version, } else if (version < version5_2) { return encodeMessage(_serializer5_1, routable, message, version5_1, version); } else { - if (_activateBucketSpaceSerialization) { + if (_configForcedBucketSpaceSerialization) { return encodeMessage(_serializer6_0, routable, message, version6_0, version); } else { if (version < version6_0) { @@ -184,7 +184,7 @@ StorageProtocol::decode(const vespalib::Version & version, } else if (version < version5_2) { return decodeMessage(_serializer5_1, data, type, version5_1, version); } else { - if (_activateBucketSpaceSerialization) { + if (_configForcedBucketSpaceSerialization) { return decodeMessage(_serializer6_0, data, type, version6_0, version); } else { if (version < version6_0) { diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h index d85e9d55d1a..56f271db1d0 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h @@ -29,7 +29,7 @@ private: ProtocolSerialization5_1 _serializer5_1; ProtocolSerialization5_2 _serializer5_2; ProtocolSerialization6_0 _serializer6_0; - bool _activateBucketSpaceSerialization; + bool _configForcedBucketSpaceSerialization; }; } diff --git a/storageserver/src/tests/testhelper.cpp b/storageserver/src/tests/testhelper.cpp index b245a6500bd..5e6a71b078f 100644 --- a/storageserver/src/tests/testhelper.cpp +++ b/storageserver/src/tests/testhelper.cpp @@ -96,7 +96,11 @@ vdstestlib::DirConfig getStandardConfig(bool storagenode) { // By default, need "old" behaviour of maxconcurrent config->set("maxconcurrentvisitors_fixed", "4"); config->set("maxconcurrentvisitors_variable", "0"); - config = &dc.addConfig("stor-visitordispatcher"); + dc.addConfig("stor-visitordispatcher"); + config = &dc.addConfig("bucketspaces"); + config->set("documenttype[1]"); + config->set("documenttype[0].name", "testdoctype1"); + config->set("documenttype[0].bucketspace", "default"); addFileConfig(dc, "documenttypes", "config-doctypes.cfg"); addStorageDistributionConfig(dc); return dc; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index 1504119d9cc..12389712976 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -12,6 +12,8 @@ import com.yahoo.vespa.athenz.utils.AthenzIdentities; import java.util.Base64; +import static com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId.*; + /** * Utility class for mapping objects model types and their Jackson binding versions. * @@ -33,7 +35,7 @@ public class EntityBindingsMapper { public static VespaUniqueInstanceId toVespaUniqueInstanceId(VespaUniqueInstanceIdEntity entity) { return new VespaUniqueInstanceId( - entity.clusterIndex, entity.clusterId, entity.instance, entity.application, entity.tenant, entity.region, entity.environment); + entity.clusterIndex, entity.clusterId, entity.instance, entity.application, entity.tenant, entity.region, entity.environment, entity.type != null ? IdentityType.fromId(entity.type) : null); // TODO Remove support for legacy representation without type } public static IdentityDocument toIdentityDocument(IdentityDocumentEntity entity) { @@ -50,17 +52,22 @@ public class EntityBindingsMapper { toIdentityDocument(entity.identityDocument), entity.signature, entity.signingKeyVersion, - VespaUniqueInstanceId.fromDottedString(entity.providerUniqueId), + fromDottedString(entity.providerUniqueId), entity.dnsSuffix, (AthenzService) AthenzIdentities.from(entity.providerService), entity.ztsEndpoint, - entity.documentVersion); + entity.documentVersion, + entity.configServerHostname, + entity.instanceHostname, + entity.createdAt, + entity.ipAddresses, + entity.identityType != null ? IdentityType.fromId(entity.identityType) : null); // TODO Remove support for legacy representation without type } public static VespaUniqueInstanceIdEntity toVespaUniqueInstanceIdEntity(VespaUniqueInstanceId model) { return new VespaUniqueInstanceIdEntity( model.tenant(), model.application(), model.environment(), model.region(), - model.instance(), model.clusterId(), model.clusterIndex()); + model.instance(), model.clusterId(), model.clusterIndex(), model.type() != null ? model.type().id() : null); // TODO Remove support for legacy representation without type } public static IdentityDocumentEntity toIdentityDocumentEntity(IdentityDocument model) { @@ -84,7 +91,12 @@ public class EntityBindingsMapper { model.dnsSuffix(), model.providerService().getFullName(), model.ztsEndpoint(), - model.documentVersion()); + model.documentVersion(), + model.configServerHostname(), + model.instanceHostname(), + model.createdAt(), + model.ipAddresses(), + model.identityType() != null ? model.identityType().id() : null); // TODO Remove support for legacy representation without type } catch (JsonProcessingException e) { throw new RuntimeException(e); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java index 8da2bd0a343..82d0a3d622c 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java @@ -8,7 +8,9 @@ import java.util.Set; * The identity document that contains the instance specific information * * @author bjorncs + * @deprecated Will soon be inlined into {@link SignedIdentityDocument} */ +@Deprecated public class IdentityDocument { private final VespaUniqueInstanceId providerUniqueId; private final String configServerHostname; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityType.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityType.java new file mode 100644 index 00000000000..4ca2e34a618 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityType.java @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api; + +import java.util.Arrays; + +/** + * Represents the types of identities that the configserver can provide. + * + * @author bjorncs + */ +public enum IdentityType {TENANT("tenant"), NODE("node"); + private final String id; + + IdentityType(String id) { this.id = id; } + + public String id() { return id; } + + public static IdentityType fromId(String id) { + return Arrays.stream(values()) + .filter(v -> v.id.equals(id)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Invalid id: " + id)); + } +} + diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index d184efc0221..60be42544c7 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.yahoo.vespa.athenz.api.AthenzService; import java.net.URI; +import java.time.Instant; +import java.util.Set; /** * A signed identity document which contains a {@link IdentityDocument} @@ -22,6 +24,11 @@ public class SignedIdentityDocument { private final AthenzService providerService; private final URI ztsEndpoint; private final int documentVersion; + private final String configServerHostname; + private final String instanceHostname; + private final Instant createdAt; + private final Set<String> ipAddresses; + private final IdentityType identityType; public SignedIdentityDocument(IdentityDocument identityDocument, String signature, @@ -30,7 +37,12 @@ public class SignedIdentityDocument { String dnsSuffix, AthenzService providerService, URI ztsEndpoint, - int documentVersion) { + int documentVersion, + String configServerHostname, + String instanceHostname, + Instant createdAt, + Set<String> ipAddresses, + IdentityType identityType) { this.identityDocument = identityDocument; this.signature = signature; this.signingKeyVersion = signingKeyVersion; @@ -39,6 +51,11 @@ public class SignedIdentityDocument { this.providerService = providerService; this.ztsEndpoint = ztsEndpoint; this.documentVersion = documentVersion; + this.configServerHostname = configServerHostname; + this.instanceHostname = instanceHostname; + this.createdAt = createdAt; + this.ipAddresses = ipAddresses; + this.identityType = identityType; } public IdentityDocument identityDocument() { @@ -72,4 +89,24 @@ public class SignedIdentityDocument { public int documentVersion() { return documentVersion; } + + public String configServerHostname() { + return configServerHostname; + } + + public String instanceHostname() { + return instanceHostname; + } + + public Instant createdAt() { + return createdAt; + } + + public Set<String> ipAddresses() { + return ipAddresses; + } + + public IdentityType identityType() { + return identityType; + } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java index 5539ba53882..be94cc59691 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceId.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.athenz.identityprovider.api; import java.util.Objects; /** + * Represents the unique instance id as used in Vespa's integration with Athenz Copper Argos + * * @author bjorncs */ public class VespaUniqueInstanceId { @@ -15,6 +17,7 @@ public class VespaUniqueInstanceId { private final String tenant; private final String region; private final String environment; + private final IdentityType type; public VespaUniqueInstanceId(int clusterIndex, String clusterId, @@ -22,7 +25,8 @@ public class VespaUniqueInstanceId { String application, String tenant, String region, - String environment) { + String environment, + IdentityType type) { this.clusterIndex = clusterIndex; this.clusterId = clusterId; this.instance = instance; @@ -30,21 +34,43 @@ public class VespaUniqueInstanceId { this.tenant = tenant; this.region = region; this.environment = environment; + this.type = type; } + // TODO Remove support for legacy representation without type + @Deprecated + public VespaUniqueInstanceId(int clusterIndex, + String clusterId, + String instance, + String application, + String tenant, + String region, + String environment) { + this(clusterIndex, clusterId, instance, application, tenant, region, environment, null); + } + + + // TODO Remove support for legacy representation without type public static VespaUniqueInstanceId fromDottedString(String instanceId) { String[] tokens = instanceId.split("\\."); - if (tokens.length != 7) { + if (tokens.length != 7 && tokens.length != 8) { throw new IllegalArgumentException("Invalid instance id: " + instanceId); } return new VespaUniqueInstanceId( - Integer.parseInt(tokens[0]), tokens[1], tokens[2], tokens[3], tokens[4], tokens[5], tokens[6]); + Integer.parseInt(tokens[0]), tokens[1], tokens[2], tokens[3], tokens[4], tokens[5], tokens[6], tokens.length == 8 ? IdentityType.fromId(tokens[7]) : null); } + // TODO Remove support for legacy representation without type public String asDottedString() { - return String.format( - "%d.%s.%s.%s.%s.%s.%s", - clusterIndex, clusterId, instance, application, tenant, region, environment); + if (type != null) { + return String.format( + "%d.%s.%s.%s.%s.%s.%s.%s", + clusterIndex, clusterId, instance, application, tenant, region, environment, type.id()); + } else { + return String.format( + "%d.%s.%s.%s.%s.%s.%s", + clusterIndex, clusterId, instance, application, tenant, region, environment); + } } public int clusterIndex() { @@ -75,6 +101,8 @@ public class VespaUniqueInstanceId { return environment; } + public IdentityType type() { return type; } + @Override public String toString() { return "VespaUniqueInstanceId{" + @@ -85,6 +113,7 @@ public class VespaUniqueInstanceId { ", tenant='" + tenant + '\'' + ", region='" + region + '\'' + ", environment='" + environment + '\'' + + ", type=" + type + '}'; } @@ -99,11 +128,12 @@ public class VespaUniqueInstanceId { Objects.equals(application, that.application) && Objects.equals(tenant, that.tenant) && Objects.equals(region, that.region) && - Objects.equals(environment, that.environment); + Objects.equals(environment, that.environment) && + type == that.type; } @Override public int hashCode() { - return Objects.hash(clusterIndex, clusterId, instance, application, tenant, region, environment); + return Objects.hash(clusterIndex, clusterId, instance, application, tenant, region, environment, type); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java index 775a49349a3..fc5392411c1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentApi.java @@ -5,7 +5,6 @@ import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; /** @@ -16,11 +15,6 @@ public interface IdentityDocumentApi { @GET @Produces(MediaType.APPLICATION_JSON) - @Deprecated - SignedIdentityDocumentEntity getIdentityDocument(@QueryParam("hostname") String hostname); - - @GET - @Produces(MediaType.APPLICATION_JSON) @Path("/node/{host}") SignedIdentityDocumentEntity getNodeIdentityDocument(@PathParam("host") String host); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java index 58a4f1e24bf..b4b2e82ab0e 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java @@ -10,8 +10,10 @@ import java.util.Set; /** * @author bjorncs + * @deprecated Will soon be inlined into {@link SignedIdentityDocumentEntity} */ @JsonIgnoreProperties(ignoreUnknown = true) +@Deprecated public class IdentityDocumentEntity { @JsonProperty("provider-unique-id") diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index e397b81ef9e..aa514b3caf3 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -11,8 +11,10 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; +import java.time.Instant; import java.util.Base64; import java.util.Objects; +import java.util.Set; /** * @author bjorncs @@ -31,6 +33,11 @@ public class SignedIdentityDocumentEntity { @JsonProperty("provider-service") public final String providerService; @JsonProperty("zts-endpoint") public final URI ztsEndpoint; @JsonProperty("document-version") public final int documentVersion; + @JsonProperty("configserver-hostname") public final String configServerHostname; + @JsonProperty("instance-hostname") public final String instanceHostname; + @JsonProperty("created-at") public final Instant createdAt; + @JsonProperty("ip-addresses") public final Set<String> ipAddresses; + @JsonProperty("identity-type") public final String identityType; @JsonCreator public SignedIdentityDocumentEntity(@JsonProperty("identity-document") String rawIdentityDocument, @@ -40,7 +47,12 @@ public class SignedIdentityDocumentEntity { @JsonProperty("dns-suffix") String dnsSuffix, @JsonProperty("provider-service") String providerService, @JsonProperty("zts-endpoint") URI ztsEndpoint, - @JsonProperty("document-version") int documentVersion) { + @JsonProperty("document-version") int documentVersion, + @JsonProperty("configserver-hostname") String configServerHostname, + @JsonProperty("instance-hostname") String instanceHostname, + @JsonProperty("created-at") Instant createdAt, + @JsonProperty("ip-addresses") Set<String> ipAddresses, + @JsonProperty("identity-type") String identityType) { this.rawIdentityDocument = rawIdentityDocument; this.identityDocument = parseIdentityDocument(rawIdentityDocument); this.signature = signature; @@ -50,6 +62,11 @@ public class SignedIdentityDocumentEntity { this.providerService = providerService; this.ztsEndpoint = ztsEndpoint; this.documentVersion = documentVersion; + this.configServerHostname = configServerHostname; + this.instanceHostname = instanceHostname; + this.createdAt = createdAt; + this.ipAddresses = ipAddresses; + this.identityType = identityType; } private static IdentityDocumentEntity parseIdentityDocument(String rawIdentityDocument) { @@ -73,7 +90,16 @@ public class SignedIdentityDocumentEntity { ", identityDocument=" + identityDocument + ", signature='" + signature + '\'' + ", signingKeyVersion=" + signingKeyVersion + + ", providerUniqueId='" + providerUniqueId + '\'' + + ", dnsSuffix='" + dnsSuffix + '\'' + + ", providerService='" + providerService + '\'' + + ", ztsEndpoint=" + ztsEndpoint + ", documentVersion=" + documentVersion + + ", configServerHostname='" + configServerHostname + '\'' + + ", instanceHostname='" + instanceHostname + '\'' + + ", createdAt=" + createdAt + + ", ipAddresses=" + ipAddresses + + ", identityType=" + identityType + '}'; } @@ -86,11 +112,20 @@ public class SignedIdentityDocumentEntity { documentVersion == that.documentVersion && Objects.equals(rawIdentityDocument, that.rawIdentityDocument) && Objects.equals(identityDocument, that.identityDocument) && - Objects.equals(signature, that.signature); + Objects.equals(signature, that.signature) && + Objects.equals(providerUniqueId, that.providerUniqueId) && + Objects.equals(dnsSuffix, that.dnsSuffix) && + Objects.equals(providerService, that.providerService) && + Objects.equals(ztsEndpoint, that.ztsEndpoint) && + Objects.equals(configServerHostname, that.configServerHostname) && + Objects.equals(instanceHostname, that.instanceHostname) && + Objects.equals(createdAt, that.createdAt) && + Objects.equals(ipAddresses, that.ipAddresses) && + Objects.equals(identityType, identityType); } @Override public int hashCode() { - return Objects.hash(rawIdentityDocument, identityDocument, signature, signingKeyVersion, documentVersion); + return Objects.hash(rawIdentityDocument, identityDocument, signature, signingKeyVersion, providerUniqueId, dnsSuffix, providerService, ztsEndpoint, documentVersion, configServerHostname, instanceHostname, createdAt, ipAddresses, identityType); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java index 3127752ac7d..103c087638d 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/VespaUniqueInstanceIdEntity.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api.bindings; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; @@ -24,14 +25,18 @@ public class VespaUniqueInstanceIdEntity { public final String clusterId; @JsonProperty("cluster-index") public final int clusterIndex; + @JsonProperty("type") + public final String type; + @JsonCreator public VespaUniqueInstanceIdEntity(@JsonProperty("tenant") String tenant, @JsonProperty("application") String application, @JsonProperty("environment") String environment, @JsonProperty("region") String region, @JsonProperty("instance") String instance, @JsonProperty("cluster-id") String clusterId, - @JsonProperty("cluster-index") int clusterIndex) { + @JsonProperty("cluster-index") int clusterIndex, + @JsonProperty("type") String type) { this.tenant = tenant; this.application = application; this.environment = environment; @@ -39,8 +44,21 @@ public class VespaUniqueInstanceIdEntity { this.instance = instance; this.clusterId = clusterId; this.clusterIndex = clusterIndex; + this.type = type; } + @Deprecated + public VespaUniqueInstanceIdEntity(String tenant, + String application, + String environment, + String region, + String instance, + String clusterId, + int clusterIndex) { + this(tenant, application, environment, region, instance, clusterId, clusterIndex, null); + } + + @Override public String toString() { return "VespaUniqueInstanceIdEntity{" + @@ -51,6 +69,7 @@ public class VespaUniqueInstanceIdEntity { ", instance='" + instance + '\'' + ", clusterId='" + clusterId + '\'' + ", clusterIndex=" + clusterIndex + + ", type='" + type + '\'' + '}'; } @@ -65,11 +84,12 @@ public class VespaUniqueInstanceIdEntity { Objects.equals(environment, that.environment) && Objects.equals(region, that.region) && Objects.equals(instance, that.instance) && - Objects.equals(clusterId, that.clusterId); + Objects.equals(clusterId, that.clusterId) && + Objects.equals(type, that.type); } @Override public int hashCode() { - return Objects.hash(tenant, application, environment, region, instance, clusterId, clusterIndex); + return Objects.hash(tenant, application, environment, region, instance, clusterId, clusterIndex, type); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 96e93ca419d..e8ef2d9f97e 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; @@ -28,7 +29,7 @@ import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS; */ class AthenzCredentialsService { - private static final ObjectMapper mapper = new ObjectMapper(); + private static final ObjectMapper mapper = new ObjectMapper().registerModule(new JavaTimeModule()); private final IdentityConfig identityConfig; private final IdentityDocumentClient identityDocumentClient; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java index 90d1312c9f9..f92956f7961 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java @@ -2,14 +2,11 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; @@ -82,15 +79,7 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { String responseContent = EntityUtils.toString(response.getEntity()); if (HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) { SignedIdentityDocumentEntity entity = objectMapper.readValue(responseContent, SignedIdentityDocumentEntity.class); - return new SignedIdentityDocument( - EntityBindingsMapper.toIdentityDocument(entity.identityDocument), - entity.signature, - entity.signingKeyVersion, - VespaUniqueInstanceId.fromDottedString(entity.providerUniqueId), - entity.dnsSuffix, - (AthenzService) AthenzIdentities.from(entity.providerService), - entity.ztsEndpoint, - entity.documentVersion); + return EntityBindingsMapper.toSignedIdentityDocument(entity); } else { throw new RuntimeException( String.format( diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java index 8c4e4c1262d..86b6c566987 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/VespaUniqueInstanceIdTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.identityprovider.api; import org.junit.Test; +import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.*; import static org.junit.Assert.*; /** @@ -12,6 +13,18 @@ public class VespaUniqueInstanceIdTest { @Test public void can_serialize_to_and_deserialize_from_string() { VespaUniqueInstanceId id = + new VespaUniqueInstanceId(1, "cluster-id", "instance", "application", "tenant", "region", "environment", TENANT); + String stringRepresentation = id.asDottedString(); + String expectedStringRepresentation = "1.cluster-id.instance.application.tenant.region.environment.tenant"; + assertEquals(expectedStringRepresentation, stringRepresentation); + VespaUniqueInstanceId deserializedId = VespaUniqueInstanceId.fromDottedString(stringRepresentation); + assertEquals(id, deserializedId); + } + + // TODO Remove support for legacy representation without type + @Test + public void supports_legacy_representation_without_type() { + VespaUniqueInstanceId id = new VespaUniqueInstanceId(1, "cluster-id", "instance", "application", "tenant", "region", "environment"); String stringRepresentation = id.asDottedString(); String expectedStringRepresentation = "1.cluster-id.instance.application.tenant.region.environment"; diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java index 2e9b29f5327..7ad465a7d80 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java @@ -11,6 +11,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.tls.KeyStoreBuilder; @@ -132,7 +133,7 @@ public class AthenzIdentityProviderImplTest { } private static String getIdentityDocument() throws JsonProcessingException { - VespaUniqueInstanceId instanceId = new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", "us-north-1", "dev"); + VespaUniqueInstanceId instanceId = new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", "us-north-1", "dev", IdentityType.TENANT); SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( new IdentityDocument(instanceId, "localhost", "x.y.com", Instant.EPOCH, Collections.emptySet()), "dummysignature", @@ -141,7 +142,12 @@ public class AthenzIdentityProviderImplTest { "dev-us-north-1.vespa.cloud", new AthenzService("vespa.vespa.provider_dev_us-north-1"), URI.create("https://zts:4443/zts/v1"), - 1); + 1, + "localhost", + "x.y.com", + Instant.EPOCH, + Collections.emptySet(), + IdentityType.TENANT); return new ObjectMapper().registerModule(new JavaTimeModule()) .writeValueAsString(EntityBindingsMapper.toSignedIdentityDocumentEntity(signedIdentityDocument)); diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp index 0225b788e68..ea9684efe01 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp +++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp @@ -72,7 +72,7 @@ nbostream::operator = (const nbostream & rhs) { return *this; } -nbostream::~nbostream() { } +nbostream::~nbostream() = default; void nbostream::fail(State s) { diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index dff744a0a41..01d0f91dfa3 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -186,6 +186,7 @@ struct MMapLimitAndAlignmentHash { }; using AutoAllocatorsMap = std::unordered_map<MMapLimitAndAlignment, AutoAllocator::UP, MMapLimitAndAlignmentHash>; +using AutoAllocatorsMapWithDefault = std::pair<AutoAllocatorsMap, alloc::MemoryAllocator *>; void createAlignedAutoAllocators(AutoAllocatorsMap & map, size_t mmapLimit) { for (size_t alignment : {0,0x200, 0x400, 0x1000}) { @@ -197,7 +198,8 @@ void createAlignedAutoAllocators(AutoAllocatorsMap & map, size_t mmapLimit) { } } -AutoAllocatorsMap createAutoAllocators() { +AutoAllocatorsMap +createAutoAllocators() { AutoAllocatorsMap map; map.reserve(3*5); for (size_t pages : {1,2,4,8,16}) { @@ -207,7 +209,30 @@ AutoAllocatorsMap createAutoAllocators() { return map; } -AutoAllocatorsMap _G_availableAutoAllocators = createAutoAllocators(); +MemoryAllocator & +getAutoAllocator(AutoAllocatorsMap & map, size_t mmapLimit, size_t alignment) { + MMapLimitAndAlignment key(mmapLimit, alignment); + auto found = map.find(key); + if (found == map.end()) { + throw IllegalArgumentException(make_string("We currently have no support for mmapLimit(%0lx) and alignment(%0lx)", mmapLimit, alignment)); + } + return *(found->second); +} + +MemoryAllocator & +getDefaultAutoAllocator(AutoAllocatorsMap & map) { + return getAutoAllocator(map, 1 * MemoryAllocator::HUGEPAGE_SIZE, 0); +} + +AutoAllocatorsMapWithDefault +createAutoAllocatorsWithDefault() { + AutoAllocatorsMapWithDefault tmp(createAutoAllocators(), nullptr); + tmp.second = &getDefaultAutoAllocator(tmp.first); + return tmp; +} + + +AutoAllocatorsMapWithDefault _G_availableAutoAllocators = createAutoAllocatorsWithDefault(); alloc::HeapAllocator _G_heapAllocatorDefault; alloc::AlignedHeapAllocator _G_4KalignedHeapAllocator(1024); alloc::AlignedHeapAllocator _G_1KalignedHeapAllocator(4096); @@ -216,11 +241,13 @@ alloc::MMapAllocator _G_mmapAllocatorDefault; } -MemoryAllocator & HeapAllocator::getDefault() { +MemoryAllocator & +HeapAllocator::getDefault() { return _G_heapAllocatorDefault; } -MemoryAllocator & AlignedHeapAllocator::get4K() { +MemoryAllocator & +AlignedHeapAllocator::get4K() { return _G_4KalignedHeapAllocator; } @@ -228,25 +255,23 @@ MemoryAllocator & AlignedHeapAllocator::get1K() { return _G_1KalignedHeapAllocator; } -MemoryAllocator & AlignedHeapAllocator::get512B() { +MemoryAllocator & +AlignedHeapAllocator::get512B() { return _G_512BalignedHeapAllocator; } -MemoryAllocator & MMapAllocator::getDefault() { +MemoryAllocator & +MMapAllocator::getDefault() { return _G_mmapAllocatorDefault; } -MemoryAllocator & AutoAllocator::getDefault() { - return getAllocator(1 * MemoryAllocator::HUGEPAGE_SIZE, 0); +MemoryAllocator & +AutoAllocator::getDefault() { + return *_G_availableAutoAllocators.second; } MemoryAllocator & AutoAllocator::getAllocator(size_t mmapLimit, size_t alignment) { - MMapLimitAndAlignment key(mmapLimit, alignment); - auto found = _G_availableAutoAllocators.find(key); - if (found == _G_availableAutoAllocators.end()) { - throw IllegalArgumentException(make_string("We currently have no support for mmapLimit(%0lx) and alignment(%0lx)", mmapLimit, alignment)); - } - return *(found->second); + return getAutoAllocator(_G_availableAutoAllocators.first, mmapLimit, alignment); } MemoryAllocator::PtrAndSize @@ -466,6 +491,12 @@ Alloc::allocMMap(size_t sz) } Alloc +Alloc::alloc() +{ + return Alloc(&AutoAllocator::getDefault()); +} + +Alloc Alloc::alloc(size_t sz, size_t mmapLimit, size_t alignment) { return Alloc(&AutoAllocator::getAllocator(mmapLimit, alignment), sz); diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index 2c3de92c58e..b52cace45a5 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -88,7 +88,7 @@ public: std::swap(_allocator, rhs._allocator); } Alloc create(size_t sz) const { - return Alloc(_allocator, sz); + return (sz == 0) ? Alloc(_allocator) : Alloc(_allocator, sz); } static Alloc allocAlignedHeap(size_t sz, size_t alignment); @@ -98,9 +98,11 @@ public: * Optional alignment is assumed to be <= system page size, since mmap * is always used when size is above limit. */ - static Alloc alloc(size_t sz=0, size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); + static Alloc alloc(size_t sz, size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); + static Alloc alloc(); private: Alloc(const MemoryAllocator * allocator, size_t sz) : _alloc(allocator->alloc(sz)), _allocator(allocator) { } + Alloc(const MemoryAllocator * allocator) : _alloc(nullptr, 0), _allocator(allocator) { } void clear() { _alloc.first = nullptr; _alloc.second = 0; |