diff options
author | Jon Bratseth <bratseth@gmail.com> | 2022-07-04 22:47:21 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2022-07-04 22:47:21 +0200 |
commit | 40744a1a30cccff9c4b4b9f67ae6891956509ee3 (patch) | |
tree | 2b904bcf1826f6748abbbb6d5d392a5a2e729570 | |
parent | bdca3da122423f78c7dace3ab10c27c575b9fa65 (diff) |
Add option to ignore undefined fields
80 files changed, 242 insertions, 139 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java index d98709569b1..894415091e4 100644 --- a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java @@ -107,7 +107,7 @@ public class ApplicationBuilder { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); } - /** normal constructor */ + /** Regular constructor */ public ApplicationBuilder(ApplicationPackage app, FileRegistry fileRegistry, DeployLogger deployLogger, diff --git a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java index 8bb9bca3249..139d20aac82 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java @@ -26,7 +26,7 @@ public class IntermediateCollection { private final DeployLogger deployLogger; private final ModelContext.Properties modelProperties; - private Map<String, ParsedSchema> parsedSchemas = new LinkedHashMap<>(); + private final Map<String, ParsedSchema> parsedSchemas = new LinkedHashMap<>(); IntermediateCollection() { this.deployLogger = new BaseDeployLogger(); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index e54ed1f1a8b..c6fcd4c115c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -36,14 +36,13 @@ public class DocumentManager { private boolean useV8GeoPositions = false; - public DocumentManager useV8GeoPositions(boolean value) { - this.useV8GeoPositions = value; + public DocumentManager useV8GeoPositions(boolean useV8GeoPositions) { + this.useV8GeoPositions = useV8GeoPositions; return this; } public DocumentmanagerConfig.Builder produce(DocumentModel model, - DocumentmanagerConfig.Builder documentConfigBuilder) - { + DocumentmanagerConfig.Builder documentConfigBuilder) { return produceDocTypes(model, documentConfigBuilder); } @@ -75,6 +74,7 @@ public class DocumentManager { } static private class IdxMap { + private final Map<Integer, Boolean> doneMap = new HashMap<>(); private final Map<String, Integer> map = new HashMap<>(); private final DataTypeRecognizer recognizer = new DataTypeRecognizer(); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index 549b94adc02..d62270034f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -33,6 +33,7 @@ import java.util.Set; * @author baldersheim */ public class DocumentTypes { + private boolean useV8GeoPositions = false; public DocumentTypes useV8GeoPositions(boolean value) { diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java index 15599c567ab..76f0bbd854e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java @@ -4,20 +4,18 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.documentmodel.DocumentTypeRepo; /** - * DocumentModel represents everything derived from a set of search definitions. - * It contains a document manager managing all defined document types. - * It contains a search manager managing all specified search definitions. - * It contains a storage manager managing all specified storage definitions. + * DocumentModel represents everything derived from a set of schemas. + * It contains a document manager managing all defined document types, and + * a search manager managing all search aspects of the schemas. * * @author baldersheim */ public class DocumentModel { - private final DocumentTypeRepo documentMan = new DocumentTypeRepo(); - private final SearchManager searchMan = new SearchManager(); + private final DocumentTypeRepo documentManager = new DocumentTypeRepo(); + private final SearchManager searchManager = new SearchManager(); - public DocumentTypeRepo getDocumentManager() { return documentMan; } - - public SearchManager getSearchManager() { return searchMan; } + public DocumentTypeRepo getDocumentManager() { return documentManager; } + public SearchManager getSearchManager() { return searchManager; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 8534e1f65a7..15a0e060e1c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -22,23 +22,26 @@ public class ContainerDocumentApi { public static final String DOCUMENT_V1_PREFIX = "/document/v1"; - public ContainerDocumentApi(ContainerCluster<?> cluster, Options options) { - addRestApiHandler(cluster, options); - addFeedHandler(cluster, options); + private final boolean ignoreUndefinedFields; + + public ContainerDocumentApi(ContainerCluster<?> cluster, HandlerOptions handlerOptions, boolean ignoreUndefinedFields) { + this.ignoreUndefinedFields = ignoreUndefinedFields; + addRestApiHandler(cluster, handlerOptions); + addFeedHandler(cluster, handlerOptions); } - private static void addFeedHandler(ContainerCluster<?> cluster, Options options) { + private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"; - var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, options); + var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, handlerOptions); cluster.addComponent(handler); - var executor = new Threadpool("feedapi-handler", cluster, options.feedApiThreadpoolOptions); + var executor = new Threadpool("feedapi-handler", handlerOptions.feedApiThreadpoolOptions); handler.inject(executor); handler.addComponent(executor); } - private static void addRestApiHandler(ContainerCluster<?> cluster, Options options) { - var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", DOCUMENT_V1_PREFIX + "/*", options); + private static void addRestApiHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { + var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", DOCUMENT_V1_PREFIX + "/*", handlerOptions); cluster.addComponent(handler); // We need to include a dummy implementation of the previous restapi handler (using the same class name). @@ -47,17 +50,19 @@ public class ContainerDocumentApi { cluster.addComponent(oldHandlerDummy); } + public boolean ignoreUndefinedFields() { return ignoreUndefinedFields; } + private static Handler<AbstractConfigProducer<?>> newVespaClientHandler( String componentId, String bindingSuffix, - Options options) { + HandlerOptions handlerOptions) { Handler<AbstractConfigProducer<?>> handler = handlerComponentSpecification(componentId); - if (options.bindings.isEmpty()) { + if (handlerOptions.bindings.isEmpty()) { handler.addServerBindings( SystemBindingPattern.fromHttpPath(bindingSuffix), SystemBindingPattern.fromHttpPath(bindingSuffix + '/')); } else { - for (String rootBinding : options.bindings) { + for (String rootBinding : handlerOptions.bindings) { String pathWithoutLeadingSlash = bindingSuffix.substring(1); handler.addServerBindings( UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash), @@ -72,11 +77,12 @@ public class ContainerDocumentApi { BundleInstantiationSpecification.getFromStrings(className, null, "vespaclient-container-plugin"), "")); } - public static final class Options { + public static final class HandlerOptions { + private final Collection<String> bindings; private final ContainerThreadpool.UserOptions feedApiThreadpoolOptions; - public Options(Collection<String> bindings, ContainerThreadpool.UserOptions feedApiThreadpoolOptions) { + public HandlerOptions(Collection<String> bindings, ContainerThreadpool.UserOptions feedApiThreadpoolOptions) { this.bindings = Collections.unmodifiableCollection(bindings); this.feedApiThreadpoolOptions = feedApiThreadpoolOptions; } @@ -84,13 +90,8 @@ public class ContainerDocumentApi { private static class Threadpool extends ContainerThreadpool { - private final ContainerCluster<?> cluster; - - Threadpool(String name, - ContainerCluster<?> cluster, - ContainerThreadpool.UserOptions threadpoolOptions) { + Threadpool(String name, ContainerThreadpool.UserOptions threadpoolOptions) { super(name, threadpoolOptions); - this.cluster = cluster; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 3d4b6dcbe7e..5adea6d50e8 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -105,7 +105,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> /** * URI prefix used for internal, usually programmatic, APIs. URIs using this - * prefix should never considered available for direct use by customers, and + * prefix should never be considered available for direct use by customers, and * normal compatibility concerns only applies to libraries using the URIs in * question, not contents served from the URIs themselves. */ @@ -414,6 +414,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void getConfig(DocumentmanagerConfig.Builder builder) { if (containerDocproc != null && containerDocproc.isCompressDocuments()) builder.enablecompression(true); + if (containerDocumentApi != null) + builder.ignoreundefinedfields(containerDocumentApi.ignoreUndefinedFields()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 1dad0c7960d..7cc1109f25f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -924,8 +924,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element documentApiElement = XML.getChild(spec, "document-api"); if (documentApiElement == null) return null; - ContainerDocumentApi.Options documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); - return new ContainerDocumentApi(cluster, documentApiOptions); + ContainerDocumentApi.HandlerOptions documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); + return new ContainerDocumentApi(cluster, documentApiOptions, + documentApiElement.getAttribute("ignore-undefined-fields").equals("true")); } private ContainerDocproc buildDocproc(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index 2ef31469c45..bb1d0af1db9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -18,9 +18,8 @@ public class DocumentApiOptionsBuilder { private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); - - public static ContainerDocumentApi.Options build(Element spec) { - return new ContainerDocumentApi.Options(getBindings(spec), threadpoolOptions(spec, "http-client-api")); + public static ContainerDocumentApi.HandlerOptions build(Element spec) { + return new ContainerDocumentApi.HandlerOptions(getBindings(spec), threadpoolOptions(spec, "http-client-api")); } private static ContainerThreadpool.UserOptions threadpoolOptions(Element spec, String elementName) { diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 1ab3c9893bf..ea468438483 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -203,6 +203,7 @@ ProcessingInContainer = element processing { DocumentApi = element document-api { ServerBindings & GenericConfig* & + attribute ignore-undefined-fields { xsd:boolean }? & element abortondocumenterror { xsd:boolean }? & element retryenabled { xsd:boolean }? & element route { text }? & diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index 00920cc6d76..c471935d9da 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index 7ea2fa42a2a..d35928a7a4b 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg index 15430101553..f8c7dacd3af 100644 --- a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg +++ b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg index a4b37113ef1..bf2ae28c417 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg index f6d4269abcd..7f93eea8e90 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index 31f514b16cd..ecb499b5ed3 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg index 1582c6572da..6073faca616 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg index 19bcb81db38..92bde94f54a 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false documenttype[].id 2987301 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg index 4222229cdd3..5ea07de4124 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg index 0d3802bcbe3..f31e5f6e7c0 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/derived/advanced/documentmanager.cfg b/config-model/src/test/derived/advanced/documentmanager.cfg index 6eea5ae038b..1ad50f57e93 100644 --- a/config-model/src/test/derived/advanced/documentmanager.cfg +++ b/config-model/src/test/derived/advanced/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg index f15b49c6b76..1ca66a7aea2 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg index 4098c0fe53c..8dc07ae8eab 100644 --- a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg index b67f1df3b4b..67c47032995 100644 --- a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg index df8d038fe79..dbe6054ce6c 100644 --- a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg +++ b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/annotationsreference/documentmanager.cfg b/config-model/src/test/derived/annotationsreference/documentmanager.cfg index 94258fdb798..d27ff5c9d07 100644 --- a/config-model/src/test/derived/annotationsreference/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsreference/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/annotationssimple/documentmanager.cfg b/config-model/src/test/derived/annotationssimple/documentmanager.cfg index e863e305643..a52837c5c06 100644 --- a/config-model/src/test/derived/annotationssimple/documentmanager.cfg +++ b/config-model/src/test/derived/annotationssimple/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg index 62084994721..4ab3f376d9d 100644 --- a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg index 640655c53b6..c2078dfa671 100644 --- a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/arrays/documentmanager.cfg b/config-model/src/test/derived/arrays/documentmanager.cfg index 9bf53699140..836f3903079 100644 --- a/config-model/src/test/derived/arrays/documentmanager.cfg +++ b/config-model/src/test/derived/arrays/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg index 9d91ad3f5ab..a5c063108e4 100644 --- a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg +++ b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/complex/documentmanager.cfg b/config-model/src/test/derived/complex/documentmanager.cfg index da4f2fc0942..dd0b7095d93 100644 --- a/config-model/src/test/derived/complex/documentmanager.cfg +++ b/config-model/src/test/derived/complex/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/declstruct/documentmanager.cfg b/config-model/src/test/derived/declstruct/documentmanager.cfg index 0286af0eade..992d210dbe2 100644 --- a/config-model/src/test/derived/declstruct/documentmanager.cfg +++ b/config-model/src/test/derived/declstruct/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/duplicate_struct/documentmanager.cfg b/config-model/src/test/derived/duplicate_struct/documentmanager.cfg index bb791943d41..4742a75205a 100644 --- a/config-model/src/test/derived/duplicate_struct/documentmanager.cfg +++ b/config-model/src/test/derived/duplicate_struct/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/duplicate_struct/documenttypes.cfg b/config-model/src/test/derived/duplicate_struct/documenttypes.cfg index 13567666674..45c1bd8700f 100644 --- a/config-model/src/test/derived/duplicate_struct/documenttypes.cfg +++ b/config-model/src/test/derived/duplicate_struct/documenttypes.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/emptydefault/documentmanager.cfg b/config-model/src/test/derived/emptydefault/documentmanager.cfg index a6530d49507..bf2a39df89f 100644 --- a/config-model/src/test/derived/emptydefault/documentmanager.cfg +++ b/config-model/src/test/derived/emptydefault/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/id/documentmanager.cfg b/config-model/src/test/derived/id/documentmanager.cfg index fafea7dd4bc..33cdbe5b996 100644 --- a/config-model/src/test/derived/id/documentmanager.cfg +++ b/config-model/src/test/derived/id/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg index b15734169a5..c6cd1a2949d 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/indexswitches/documentmanager.cfg b/config-model/src/test/derived/indexswitches/documentmanager.cfg index 7f6b9150e07..fa91cb0e554 100644 --- a/config-model/src/test/derived/indexswitches/documentmanager.cfg +++ b/config-model/src/test/derived/indexswitches/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/inheritance/documentmanager.cfg b/config-model/src/test/derived/inheritance/documentmanager.cfg index f0917864ebd..52cac7bfa79 100644 --- a/config-model/src/test/derived/inheritance/documentmanager.cfg +++ b/config-model/src/test/derived/inheritance/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg index 10fdcf50026..7d5dacbe00f 100644 --- a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg +++ b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg index 8c0ffd50840..537c452c38c 100644 --- a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg index 186f3c0e3c1..d246e886a3d 100644 --- a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg index e8fd97671ff..efd8170d95e 100644 --- a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg +++ b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/multi_struct/documentmanager.cfg b/config-model/src/test/derived/multi_struct/documentmanager.cfg index dbf753c6bc3..e37a3dc51c6 100644 --- a/config-model/src/test/derived/multi_struct/documentmanager.cfg +++ b/config-model/src/test/derived/multi_struct/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/multi_struct/documenttypes.cfg b/config-model/src/test/derived/multi_struct/documenttypes.cfg index 9ce19079177..93452602f86 100644 --- a/config-model/src/test/derived/multi_struct/documenttypes.cfg +++ b/config-model/src/test/derived/multi_struct/documenttypes.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/namecollision/documentmanager.cfg b/config-model/src/test/derived/namecollision/documentmanager.cfg index 73730595574..d8cf44a9a3d 100644 --- a/config-model/src/test/derived/namecollision/documentmanager.cfg +++ b/config-model/src/test/derived/namecollision/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg index be048ee2eba..d516eaf7886 100644 --- a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg +++ b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/ranktypes/documentmanager.cfg b/config-model/src/test/derived/ranktypes/documentmanager.cfg index 33d41084651..46457fb479d 100644 --- a/config-model/src/test/derived/ranktypes/documentmanager.cfg +++ b/config-model/src/test/derived/ranktypes/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/reference_from_several/documentmanager.cfg b/config-model/src/test/derived/reference_from_several/documentmanager.cfg index 6ac1aab0baf..28f40aeee5b 100644 --- a/config-model/src/test/derived/reference_from_several/documentmanager.cfg +++ b/config-model/src/test/derived/reference_from_several/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/schemainheritance/documentmanager.cfg b/config-model/src/test/derived/schemainheritance/documentmanager.cfg index 12c29667079..1fe61cf2bd2 100644 --- a/config-model/src/test/derived/schemainheritance/documentmanager.cfg +++ b/config-model/src/test/derived/schemainheritance/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index 81de2d7ebdb..b94f23a9c7b 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/structandfieldset/documentmanager.cfg b/config-model/src/test/derived/structandfieldset/documentmanager.cfg index 85df5249e3a..e1169e3ca5d 100644 --- a/config-model/src/test/derived/structandfieldset/documentmanager.cfg +++ b/config-model/src/test/derived/structandfieldset/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/structanyorder/documentmanager.cfg b/config-model/src/test/derived/structanyorder/documentmanager.cfg index b3b6a2587c1..eac63515944 100644 --- a/config-model/src/test/derived/structanyorder/documentmanager.cfg +++ b/config-model/src/test/derived/structanyorder/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/structinheritance/documentmanager.cfg b/config-model/src/test/derived/structinheritance/documentmanager.cfg index ca11481afa4..37240887e3b 100644 --- a/config-model/src/test/derived/structinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/structinheritance/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/structinheritance/documenttypes.cfg b/config-model/src/test/derived/structinheritance/documenttypes.cfg index 8b343665289..16521b920a7 100644 --- a/config-model/src/test/derived/structinheritance/documenttypes.cfg +++ b/config-model/src/test/derived/structinheritance/documenttypes.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/tensor/documentmanager.cfg b/config-model/src/test/derived/tensor/documentmanager.cfg index d11c67fea01..f52fe073208 100644 --- a/config-model/src/test/derived/tensor/documentmanager.cfg +++ b/config-model/src/test/derived/tensor/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index 9ee6a82245f..3081a5bd0c4 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index 9a73664ac86..118f7d279e6 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[].name "document" diff --git a/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg b/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg index ea56800ea1c..2efc2f40d21 100644 --- a/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg +++ b/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index b59b3206a2d..10c66ce3e93 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/examples/structresult.cfg b/config-model/src/test/examples/structresult.cfg index 34383084625..2e3904b7110 100644 --- a/config-model/src/test/examples/structresult.cfg +++ b/config-model/src/test/examples/structresult.cfg @@ -1,3 +1,4 @@ +ignoreundefinedfields false enablecompression false usev8geopositions false doctype[0].name "document" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index b4242336c5c..ca0b4681e51 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -5,7 +5,9 @@ import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; @@ -63,7 +65,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa } @Test - public void requireThatHandlersAreSetup() { + public void test_handler_setup() { Element elem = DomBuilderTest.parse( "<container id='cluster1' version='1.0'>", " <document-api />", @@ -86,6 +88,20 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa } @Test + public void nonexisting_fields_can_be_ignored() { + Element elem = DomBuilderTest.parse( + "<container id='cluster1' version='1.0'>", + " <document-api ignore-undefined-fields='true' />", + nodesXml, + "</container>"); + ContainerModel model = createModel(root, elem).get(0); + + var documentManager = new DocumentmanagerConfig.Builder(); + model.getCluster().getConfig(documentManager); + assertTrue(documentManager.build().ignoreundefinedfields()); + } + + @Test public void feeding_api_have_separate_threadpools() { Element elem = DomBuilderTest.parse( "<container id='cluster1' version='1.0'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java index cfe4b72f37d..7e8852ce85a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java @@ -56,18 +56,21 @@ public abstract class ContainerModelBuilderTestBase { createModel(root, clusterElem); } - public static void createModel(MockRoot root, DeployState deployState, VespaModel vespaModel, Element... containerElems) { + public static List<ContainerModel> createModel(MockRoot root, DeployState deployState, VespaModel vespaModel, Element... containerElems) { + List<ContainerModel> containerModels = new ArrayList<>(); for (Element containerElem : containerElems) { ContainerModel model = new ContainerModelBuilder(false, ContainerModelBuilder.Networking.enable) .build(deployState, vespaModel, null, root, containerElem); ContainerCluster<?> cluster = model.getCluster(); generateDefaultSearchChains(cluster); + containerModels.add(model); } root.freezeModelTopology(); + return containerModels; } - public static void createModel(MockRoot root, Element... containerElems) { - createModel(root, DeployState.createTestState(), null, containerElems); + public static List<ContainerModel> createModel(MockRoot root, Element... containerElems) { + return createModel(root, DeployState.createTestState(), null, containerElems); } public static void createModel(MockRoot root, DeployLogger testLogger, Element... containerElems) { diff --git a/document/abi-spec.json b/document/abi-spec.json index 0ddc5dabc3c..443385bb0a0 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -208,7 +208,7 @@ "public bridge synthetic int compareTo(java.lang.Object)" ], "fields": [ - "public static int classId", + "public static final int classId", "public static final com.yahoo.document.NumericDataType NONE", "public static final com.yahoo.document.NumericDataType INT", "public static final com.yahoo.document.NumericDataType FLOAT", @@ -485,6 +485,8 @@ "public void <init>()", "public void <init>(com.yahoo.document.config.DocumentmanagerConfig)", "public static com.yahoo.document.DocumentTypeManager fromFile(java.lang.String)", + "public boolean getIgnoreUndefinedFields()", + "public void setIgnoreUndefinedFields(boolean)", "public void register(com.yahoo.document.DataType)", "public com.yahoo.document.DocumentType registerDocumentType(com.yahoo.document.DocumentType)", "public com.yahoo.document.DocumentType getDocumentType(com.yahoo.document.DataTypeName)", diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index b185ff53c87..e47c0c3a81e 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -34,7 +34,7 @@ import java.util.List; public abstract class DataType extends Identifiable implements Comparable<DataType> { // The global class identifier shared with C++. - public static int classId = registerClass(Ids.document + 50, DataType.class); + public static final int classId = registerClass(Ids.document + 50, DataType.class); // NOTE: These types are also defined in // document/src/vespa/document/datatype/datatype.h diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java index b5c9e54939e..379e0836a01 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -2,7 +2,6 @@ package com.yahoo.document; import com.yahoo.component.annotation.Inject; -import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.document.annotation.AnnotationTypes; @@ -21,12 +20,11 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.logging.Logger; /** * The DocumentTypeManager keeps track of the document types registered in * the Vespa common repository. - * <p> + * * The DocumentTypeManager is also responsible for registering a FieldValue * factory for each data type a field can have. The Document object * uses this factory to serialize and deserialize the various datatypes. @@ -38,9 +36,6 @@ import java.util.logging.Logger; */ public class DocumentTypeManager { - private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName()); - - // *Configured data types* (not built-in/primitive) indexed by their id // // *Primitive* data types are always available and have a single id. @@ -54,6 +49,7 @@ public class DocumentTypeManager { private Map<Integer, DataType> dataTypes = new LinkedHashMap<>(); private Map<DataTypeName, DocumentType> documentTypes = new LinkedHashMap<>(); private AnnotationTypeRegistry annotationTypeRegistry = new AnnotationTypeRegistry(); + private boolean ignoreUndefinedFields = false; public DocumentTypeManager() { registerDefaultDataTypes(); @@ -103,22 +99,12 @@ public class DocumentTypeManager { } } - boolean hasDataTypeInternal(String name) { - if (name.startsWith("tensor(")) return true; // built-in dynamic: Always present - for (DataType type : dataTypes.values()) { - if (type.getName().equalsIgnoreCase(name)) { - return true; - } - } - return false; - } - /** * For internal use only, avoid whenever possible. * Use constants and factories in DataType instead. * For structs, use getStructType() in DocumentType. * For annotation payloads, use getDataType() in AnnotationType. - **/ + */ DataType getDataTypeInternal(String name) { if (name.startsWith("tensor(")) // built-in dynamic return new TensorDataType(TensorType.fromSpec(name)); @@ -151,6 +137,13 @@ public class DocumentTypeManager { } /** + * Returns true if we should ignore attempts to set a field not defined in the document type, + * rather than (by default) throwing an exception. + */ + public boolean getIgnoreUndefinedFields() { return ignoreUndefinedFields; } + public void setIgnoreUndefinedFields(boolean ignoreUndefinedFields) { this.ignoreUndefinedFields = ignoreUndefinedFields; } + + /** * Return a data type instance * * @param code the code of the data type to return, which must be either built in or present in this manager diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index d6833a482f1..518a80ca058 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -60,9 +60,8 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub /** One-shot configuration; should be called on a newly constructed manager */ static void configureNewManager(DocumentmanagerConfig config, DocumentTypeManager manager) { - if (config == null) { - return; - } + if (config == null) return; + manager.setIgnoreUndefinedFields(config.ignoreundefinedfields()); new Apply(config, manager); if (config.datatype().size() == 0 && config.annotationtype().size() == 0) { new ApplyNewDoctypeConfig(config, manager); @@ -328,6 +327,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub public ApplyNewDoctypeConfig(DocumentmanagerConfig config, DocumentTypeManager manager) { this.manager = manager; this.usev8geopositions = config.usev8geopositions(); + apply(config); } diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 009fcb3de28..94ce986fc81 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -71,7 +71,7 @@ public class JsonReader { throw new IllegalArgumentException(e); } documentParseInfo.operationType = operationType; - VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(); + VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); DocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation( getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo); operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition)); @@ -103,7 +103,7 @@ public class JsonReader { state = END_OF_FEED; return null; } - VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(); + VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); DocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation( getDocumentTypeFromString(documentParseInfo.get().documentId.getDocType(), typeManager), documentParseInfo.get()); diff --git a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java index b0882c3ab03..bc214f18776 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java +++ b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java @@ -21,22 +21,22 @@ public class AddRemoveCreator { // yes, this suppresswarnings ugliness is by intention, the code relies on // the contracts in the builders - @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) - public static void createAdds(TokenBuffer buffer, Field field, FieldUpdate update) { - createAddsOrRemoves(buffer, field, update, false); + @SuppressWarnings("cast") + public static void createAdds(TokenBuffer buffer, Field field, FieldUpdate update, boolean ignoreUndefinedFields) { + createAddsOrRemoves(buffer, field, update, false, ignoreUndefinedFields); } // yes, this suppresswarnings ugliness is by intention, the code relies on // the contracts in the builders - @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) - public static void createRemoves(TokenBuffer buffer, Field field, FieldUpdate update) { - createAddsOrRemoves(buffer, field, update, true); + @SuppressWarnings("cast") + public static void createRemoves(TokenBuffer buffer, Field field, FieldUpdate update, boolean ignoreUndefinedFields) { + createAddsOrRemoves(buffer, field, update, true, ignoreUndefinedFields); } // yes, this suppresswarnings ugliness is by intention, the code relies on // the contracts in the builders @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) - private static void createAddsOrRemoves(TokenBuffer buffer, Field field, FieldUpdate update, boolean isRemove) { + private static void createAddsOrRemoves(TokenBuffer buffer, Field field, FieldUpdate update, boolean isRemove, boolean ignoreUndefinedFields) { FieldValue container = field.getDataType().createFieldValue(); FieldUpdate singleUpdate; int initNesting = buffer.nesting(); @@ -45,10 +45,9 @@ public class AddRemoveCreator { if (container instanceof CollectionFieldValue) { buffer.next(); DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType(); - if (container instanceof WeightedSet) { + if (container instanceof WeightedSet weightedSet) { // these are objects with string keys (which are the nested // types) and values which are the weight - WeightedSet weightedSet = (WeightedSet) container; fillWeightedSetUpdate(buffer, initNesting, valueType, weightedSet); if (isRemove) { singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet); @@ -58,7 +57,7 @@ public class AddRemoveCreator { } } else { List<FieldValue> arrayContents = new ArrayList<>(); - ArrayReader.fillArrayUpdate(buffer, initNesting, valueType, arrayContents); + ArrayReader.fillArrayUpdate(buffer, initNesting, valueType, arrayContents, ignoreUndefinedFields); if (buffer.currentToken() != JsonToken.END_ARRAY) { throw new IllegalArgumentException("Expected END_ARRAY. Got '" + buffer.currentToken() + "'."); } diff --git a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java index 9c27b1eb9d0..e8c4ae85356 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java @@ -15,22 +15,23 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; public class ArrayReader { - public static void fillArrayUpdate(TokenBuffer buffer, int initNesting, DataType valueType, List<FieldValue> arrayContents) { + public static void fillArrayUpdate(TokenBuffer buffer, int initNesting, DataType valueType, + List<FieldValue> arrayContents, boolean ignoreUndefinedFields) { while (buffer.nesting() >= initNesting) { Preconditions.checkArgument(buffer.currentToken() != JsonToken.VALUE_NULL, "Illegal null value for array entry"); - arrayContents.add(readSingleValue(buffer, valueType)); + arrayContents.add(readSingleValue(buffer, valueType, ignoreUndefinedFields)); buffer.next(); } } @SuppressWarnings({ "unchecked", "rawtypes" }) - public static void fillArray(TokenBuffer buffer, CollectionFieldValue parent, DataType valueType) { + public static void fillArray(TokenBuffer buffer, CollectionFieldValue parent, DataType valueType, boolean ignoreUndefinedFields) { int initNesting = buffer.nesting(); expectArrayStart(buffer.currentToken()); buffer.next(); while (buffer.nesting() >= initNesting) { Preconditions.checkArgument(buffer.currentToken() != JsonToken.VALUE_NULL, "Illegal null value for array entry"); - parent.add(readSingleValue(buffer, valueType)); + parent.add(readSingleValue(buffer, valueType, ignoreUndefinedFields)); buffer.next(); } } diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java index 91274144710..a2dd91b90a0 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java @@ -19,10 +19,10 @@ import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSet; public class CompositeReader { - // TODO createComposite is extremely similar to add/remove, refactor + // TODO: reateComposite is extremely similar to add/remove, refactor // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders @SuppressWarnings({ "cast", "rawtypes" }) - public static void populateComposite(TokenBuffer buffer, FieldValue fieldValue) { + public static void populateComposite(TokenBuffer buffer, FieldValue fieldValue, boolean ignoreUndefinedFields) { JsonToken token = buffer.currentToken(); if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) { throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'."); @@ -32,14 +32,14 @@ public class CompositeReader { if (fieldValue instanceof WeightedSet) { fillWeightedSet(buffer, valueType, (WeightedSet) fieldValue); } else { - fillArray(buffer, (CollectionFieldValue) fieldValue, valueType); + fillArray(buffer, (CollectionFieldValue) fieldValue, valueType, ignoreUndefinedFields); } } else if (fieldValue instanceof MapFieldValue) { - MapReader.fillMap(buffer, (MapFieldValue) fieldValue); + MapReader.fillMap(buffer, (MapFieldValue) fieldValue, ignoreUndefinedFields); } else if (PositionDataType.INSTANCE.equals(fieldValue.getDataType())) { GeoPositionReader.fillGeoPosition(buffer, fieldValue); } else if (fieldValue instanceof StructuredFieldValue) { - StructReader.fillStruct(buffer, (StructuredFieldValue) fieldValue); + StructReader.fillStruct(buffer, (StructuredFieldValue) fieldValue, ignoreUndefinedFields); } else if (fieldValue instanceof TensorFieldValue) { TensorReader.fillTensor(buffer, (TensorFieldValue) fieldValue); } else { diff --git a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java index fa8f9bfd6c5..1d4cb85d130 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java @@ -25,21 +25,22 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; public class MapReader { + public static final String MAP_KEY = "key"; public static final String MAP_VALUE = "value"; public static final String UPDATE_ELEMENT = "element"; public static final String UPDATE_MATCH = "match"; - public static void fillMap(TokenBuffer buffer, MapFieldValue parent) { + public static void fillMap(TokenBuffer buffer, MapFieldValue parent, boolean ignoreUndefinedFields) { if (buffer.currentToken() == JsonToken.START_ARRAY) { - MapReader.fillMapFromArray(buffer, parent); + MapReader.fillMapFromArray(buffer, parent, ignoreUndefinedFields); } else { - MapReader.fillMapFromObject(buffer, parent); + MapReader.fillMapFromObject(buffer, parent, ignoreUndefinedFields); } } @SuppressWarnings({ "rawtypes", "cast", "unchecked" }) - public static void fillMapFromArray(TokenBuffer buffer, MapFieldValue parent) { + public static void fillMapFromArray(TokenBuffer buffer, MapFieldValue parent, boolean ignoreUndefinedFields) { JsonToken token = buffer.currentToken(); int initNesting = buffer.nesting(); expectArrayStart(token); @@ -53,9 +54,9 @@ public class MapReader { token = buffer.next(); for (int i = 0; i < 2; ++i) { if (MAP_KEY.equals(buffer.currentName())) { - key = readSingleValue(buffer, keyType); + key = readSingleValue(buffer, keyType, ignoreUndefinedFields); } else if (MAP_VALUE.equals(buffer.currentName())) { - value = readSingleValue(buffer, valueType); + value = readSingleValue(buffer, valueType, ignoreUndefinedFields); } token = buffer.next(); } @@ -68,7 +69,7 @@ public class MapReader { } @SuppressWarnings({ "rawtypes", "cast", "unchecked" }) - public static void fillMapFromObject(TokenBuffer buffer, MapFieldValue parent) { + public static void fillMapFromObject(TokenBuffer buffer, MapFieldValue parent, boolean ignoreUndefinedFields) { JsonToken token = buffer.currentToken(); int initNesting = buffer.nesting(); expectObjectStart(token); @@ -77,7 +78,7 @@ public class MapReader { DataType valueType = parent.getDataType().getValueType(); while (buffer.nesting() >= initNesting) { FieldValue key = readAtomic(buffer.currentName(), keyType); - FieldValue value = readSingleValue(buffer, valueType); + FieldValue value = readSingleValue(buffer, valueType, ignoreUndefinedFields); Preconditions.checkState(key != null && value != null, "Missing key or value for map entry."); parent.put(key, value); @@ -87,7 +88,11 @@ public class MapReader { } @SuppressWarnings({ "rawtypes", "unchecked" }) - public static ValueUpdate createMapUpdate(TokenBuffer buffer, DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) { + public static ValueUpdate createMapUpdate(TokenBuffer buffer, + DataType currentLevel, + FieldValue keyParent, + FieldValue topLevelKey, + boolean ignoreUndefinedFields) { TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT); if (UPDATE_ELEMENT.equals(buffer.currentName())) { buffer.next(); @@ -102,24 +107,24 @@ public class MapReader { if (!UPDATE_MATCH.equals(buffer.currentName())) { // we have reached an action... if (topLevelKey == null) { - return ValueUpdate.createMap(key, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName())); + return ValueUpdate.createMap(key, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName(), ignoreUndefinedFields)); } else { - return ValueUpdate.createMap(topLevelKey, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName())); + return ValueUpdate.createMap(topLevelKey, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName(), ignoreUndefinedFields)); } } else { // next level of matching if (topLevelKey == null) { - return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, key); + return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, key, ignoreUndefinedFields); } else { - return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, topLevelKey); + return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, topLevelKey, ignoreUndefinedFields); } } } @SuppressWarnings("rawtypes") - public static ValueUpdate createMapUpdate(TokenBuffer buffer, Field field) { + public static ValueUpdate createMapUpdate(TokenBuffer buffer, Field field, boolean ignoreUndefinedFields) { buffer.next(); - MapValueUpdate m = (MapValueUpdate) MapReader.createMapUpdate(buffer, field.getDataType(), null, null); + MapValueUpdate m = (MapValueUpdate) MapReader.createMapUpdate(buffer, field.getDataType(), null, null, ignoreUndefinedFields); buffer.next(); // must generate the field value in parallell with the actual return m; diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java index 0ccdbdaa9d5..1747e739bbf 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -41,25 +41,25 @@ public class SingleValueReader { arithmeticExpressionPattern = Pattern.compile("^\\$\\w+\\s*([" + validSigns + "])\\s*(\\d+(.\\d+)?)$"); } - public static FieldValue readSingleValue(TokenBuffer buffer, DataType expectedType) { + public static FieldValue readSingleValue(TokenBuffer buffer, DataType expectedType, boolean ignoreUndefinedFields) { if (buffer.currentToken().isScalarValue()) { return readAtomic(buffer.currentText(), expectedType); } else { FieldValue fieldValue = expectedType.createFieldValue(); - CompositeReader.populateComposite(buffer, fieldValue); + CompositeReader.populateComposite(buffer, fieldValue, ignoreUndefinedFields); return fieldValue; } } @SuppressWarnings("rawtypes") - public static ValueUpdate readSingleUpdate(TokenBuffer buffer, DataType expectedType, String action) { + public static ValueUpdate readSingleUpdate(TokenBuffer buffer, DataType expectedType, String action, boolean ignoreUndefinedFields) { ValueUpdate update; switch (action) { case UPDATE_ASSIGN: update = (buffer.currentToken() == JsonToken.VALUE_NULL) ? ValueUpdate.createClear() - : ValueUpdate.createAssign(readSingleValue(buffer, expectedType)); + : ValueUpdate.createAssign(readSingleValue(buffer, expectedType, ignoreUndefinedFields)); break; // double is silly, but it's what is used internally anyway case UPDATE_INCREMENT: diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java index 54591134d4c..b9eaf0d8ec6 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java @@ -12,31 +12,31 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; public class StructReader { - public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent) { + public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent, boolean ignoreUndefinedFields) { // do note the order of initializing initNesting and token is relevant for empty docs int initNesting = buffer.nesting(); buffer.next(); while (buffer.nesting() >= initNesting) { - Field f = getField(buffer, parent); + Field field = getField(buffer, parent, ignoreUndefinedFields); try { - // skip fields set to null - if (buffer.currentToken() != JsonToken.VALUE_NULL) { - FieldValue v = readSingleValue(buffer, f.getDataType()); - parent.setFieldValue(f, v); + if (field != null && buffer.currentToken() != JsonToken.VALUE_NULL) { + FieldValue v = readSingleValue(buffer, field.getDataType(), ignoreUndefinedFields); + parent.setFieldValue(field, v); } buffer.next(); } catch (IllegalArgumentException e) { - throw new JsonReaderException(f, e); + throw new JsonReaderException(field, e); } } } - public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) { + private static Field getField(TokenBuffer buffer, StructuredFieldValue parent, boolean ignoreUndefinedFields) { Field field = parent.getField(buffer.currentName()); - if (field == null) { + if (field == null && ! ignoreUndefinedFields) { throw new IllegalArgumentException("No field '" + buffer.currentName() + "' in the structure of type '" + - parent.getDataType().getDataTypeName() + "', which has the fields:" + parent.getDataType().getFields()); + parent.getDataType().getDataTypeName() + + "', which has the fields: " + parent.getDataType().getFields()); } return field; } diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java index fb160a9bb44..7bc462ec73a 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java @@ -44,6 +44,12 @@ public class VespaJsonDocumentReader { private static final String UPDATE_REMOVE = "remove"; private static final String UPDATE_ADD = "add"; + private final boolean ignoreUndefinedFields; + + public VespaJsonDocumentReader(boolean ignoreUndefinedFields) { + this.ignoreUndefinedFields = ignoreUndefinedFields; + } + public DocumentOperation createDocumentOperation(DocumentType documentType, DocumentParseInfo documentParseInfo) { final DocumentOperation documentOperation; try { @@ -82,7 +88,7 @@ public class VespaJsonDocumentReader { try { if (buffer.isEmpty()) // no "fields" map throw new IllegalArgumentException(put + " is missing a 'fields' map"); - populateComposite(buffer, put.getDocument()); + populateComposite(buffer, put.getDocument(), ignoreUndefinedFields); } catch (JsonReaderException e) { throw JsonReaderException.addDocId(e, put.getId()); } @@ -129,25 +135,25 @@ public class VespaJsonDocumentReader { if (isTensorField(field)) { fieldUpdate.addValueUpdate(createTensorRemoveUpdate(buffer, field)); } else { - createRemoves(buffer, field, fieldUpdate); + createRemoves(buffer, field, fieldUpdate, ignoreUndefinedFields); } break; case UPDATE_ADD: if (isTensorField(field)) { fieldUpdate.addValueUpdate(createTensorAddUpdate(buffer, field)); } else { - createAdds(buffer, field, fieldUpdate); + createAdds(buffer, field, fieldUpdate, ignoreUndefinedFields); } break; case UPDATE_MATCH: - fieldUpdate.addValueUpdate(createMapUpdate(buffer, field)); + fieldUpdate.addValueUpdate(createMapUpdate(buffer, field, ignoreUndefinedFields)); break; case UPDATE_MODIFY: fieldUpdate.addValueUpdate(createModifyUpdate(buffer, field)); break; default: String action = buffer.currentName(); - fieldUpdate.addValueUpdate(readSingleUpdate(buffer, field.getDataType(), action)); + fieldUpdate.addValueUpdate(readSingleUpdate(buffer, field.getDataType(), action, ignoreUndefinedFields)); } buffer.next(); } @@ -183,14 +189,16 @@ public class VespaJsonDocumentReader { private AssignFieldPathUpdate readAssignFieldPathUpdate(DocumentType documentType, String fieldPath, TokenBuffer buffer) { AssignFieldPathUpdate fieldPathUpdate = new AssignFieldPathUpdate(documentType, fieldPath); - FieldValue fv = SingleValueReader.readSingleValue(buffer, fieldPathUpdate.getFieldPath().getResultingDataType()); + FieldValue fv = SingleValueReader.readSingleValue(buffer, fieldPathUpdate.getFieldPath().getResultingDataType(), + ignoreUndefinedFields); fieldPathUpdate.setNewValue(fv); return fieldPathUpdate; } private AddFieldPathUpdate readAddFieldPathUpdate(DocumentType documentType, String fieldPath, TokenBuffer buffer) { AddFieldPathUpdate fieldPathUpdate = new AddFieldPathUpdate(documentType, fieldPath); - FieldValue fv = SingleValueReader.readSingleValue(buffer, fieldPathUpdate.getFieldPath().getResultingDataType()); + FieldValue fv = SingleValueReader.readSingleValue(buffer, fieldPathUpdate.getFieldPath().getResultingDataType(), + ignoreUndefinedFields); fieldPathUpdate.setNewValues((Array) fv); return fieldPathUpdate; } @@ -200,8 +208,8 @@ public class VespaJsonDocumentReader { return new RemoveFieldPathUpdate(documentType, fieldPath); } - private AssignFieldPathUpdate readArithmeticFieldPathUpdate( - DocumentType documentType, String fieldPath, TokenBuffer buffer, String fieldPathOperation) { + private AssignFieldPathUpdate readArithmeticFieldPathUpdate(DocumentType documentType, String fieldPath, + TokenBuffer buffer, String fieldPathOperation) { AssignFieldPathUpdate fieldPathUpdate = new AssignFieldPathUpdate(documentType, fieldPath); String arithmeticSign = SingleValueReader.UPDATE_OPERATION_TO_ARITHMETIC_SIGN.get(fieldPathOperation); double value = Double.valueOf(buffer.currentText()); @@ -217,7 +225,7 @@ public class VespaJsonDocumentReader { private static void verifyEndState(TokenBuffer buffer, JsonToken expectedFinalToken) { Preconditions.checkState(buffer.currentToken() == expectedFinalToken, - "Expected end of JSON struct (%s), got %s", expectedFinalToken, buffer.currentToken()); + "Expected end of JSON struct (%s), got %s", expectedFinalToken, buffer.currentToken()); Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation"); Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation"); Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation"); diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 675ad502630..441f1fd28ea 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -258,7 +258,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); smokeTestDoc(put.getDocument()); } @@ -277,7 +277,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); smokeTestDoc(put.getDocument()); } @@ -288,7 +288,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); assertEquals("id:unittest:smoke::whee", parseInfo.documentId.toString()); } @@ -302,7 +302,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("skuggsjaa")); assertSame(Struct.class, f.getClass()); @@ -316,7 +316,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate update = new DocumentUpdate(docType, parseInfo.documentId); - new VespaJsonDocumentReader().readUpdate(parseInfo.fieldsBuffer, update); + new VespaJsonDocumentReader(false).readUpdate(parseInfo.fieldsBuffer, update); return update; } @@ -505,7 +505,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); return put.getDocument(); } @@ -684,7 +684,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualraw")); assertSame(Raw.class, f.getClass()); @@ -701,7 +701,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue("actualMapStringToArrayOfInt"); assertSame(MapFieldValue.class, f.getClass()); @@ -722,7 +722,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue("actualMapStringToArrayOfInt"); assertSame(MapFieldValue.class, f.getClass()); @@ -987,7 +987,7 @@ public class JsonReaderTestCase { } @Test - public void misspelledFieldTest() throws IOException{ + public void nonExistingFieldCausesException() throws IOException{ JsonReader r = createReader(inputJson( "{ 'put': 'id:unittest:smoke::whee',", " 'fields': {", @@ -996,8 +996,9 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); + try { - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); fail(); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().startsWith("No field 'smething' in the structure of type 'smoke'")); @@ -1005,6 +1006,20 @@ public class JsonReaderTestCase { } @Test + public void nonExistingFieldCanBeIgnored() throws IOException{ + JsonReader r = createReader(inputJson( + "{ 'put': 'id:unittest:smoke::whee',", + " 'fields': {", + " 'smething': 'smoketest',", + " 'nalle': 'bamse' }}")); + DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentType docType = r.readDocumentType(parseInfo.documentId); + DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); + + new VespaJsonDocumentReader(true).readPut(parseInfo.fieldsBuffer, put); + } + + @Test public void feedWithBasicErrorTest() { JsonReader r = createReader(inputJson("[", " { 'put': 'id:test:smoke::0', 'fields': { 'something': 'foo' } },", @@ -1029,7 +1044,7 @@ public class JsonReaderTestCase { DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(parseInfo.fieldsBuffer, put); smokeTestDoc(put.getDocument()); } diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 92f28d8f219..e7368a691ab 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -349,7 +349,7 @@ public class JsonWriterTestCase { DocumentParseInfo raw = r.parseDocument().get(); DocumentType docType = r.readDocumentType(raw.documentId); DocumentPut put = new DocumentPut(new Document(docType, raw.documentId)); - new VespaJsonDocumentReader().readPut(raw.fieldsBuffer, put); + new VespaJsonDocumentReader(false).readPut(raw.fieldsBuffer, put); return put.getDocument(); } diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index ec19ba8d802..b1929e42d34 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -2,6 +2,9 @@ namespace=document.config +## Whether attempts to set an undefined field should be ignored rather than causing an error +ignoreundefinedfields bool default=false + ## Whether to enable compression in this process. enablecompression bool default=false diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 202447295c3..3138e71e025 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -2,6 +2,9 @@ namespace=document.config +## Whether attempts to set an undefined field should be ignored rather than causing an error +ignoreundefinedfields bool default=false + ## Whether to enable compression in this process. enablecompression bool default=false |