diff options
author | Arne H Juul <arnej@yahooinc.com> | 2021-12-08 10:28:28 +0000 |
---|---|---|
committer | Arne H Juul <arnej@yahooinc.com> | 2021-12-08 10:38:22 +0000 |
commit | 5e8a7d76623d12b9f41ad7194ba4426d1cc6254f (patch) | |
tree | 8086bd351957abcfff155f2153c6bbb546cee426 /document | |
parent | 0f5760a244f24b788c388be77973338781a3584d (diff) |
add another Apply implentation
* if there is no config at all in the old arrays,
read new per-doctype config instead
Diffstat (limited to 'document')
-rw-r--r-- | document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java | 274 |
1 files changed, 261 insertions, 13 deletions
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 1f9e494aa29..c950f6b5163 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -11,15 +11,19 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.HashSet; +import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.function.Supplier; +import com.yahoo.tensor.TensorType; /** * Configures the Vespa document manager from a config id. * * @author Einar M R Rosenvinge */ -public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig>{ +public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig> { private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName()); @@ -65,15 +69,16 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub return; } new Apply(config, manager); + if (config.datatype().size() == 0 && config.annotationtype().size() == 0) { + new ApplyNewDoctypeConfig(config, manager); + } } private static class Apply { public Apply(DocumentmanagerConfig config, DocumentTypeManager manager) { this.manager = manager; - this.usev8geopositions = (config == null) ? false : config.usev8geopositions(); - if (config != null) { - apply(config); - } + this.usev8geopositions = config.usev8geopositions(); + apply(config); } private Map<Integer, DataType> typesById = new HashMap<>(); @@ -109,15 +114,12 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub .collect(Collectors.toUnmodifiableSet()); DocumentType type = new DocumentType(doc.name(), header, importedFields); if (id != type.getId()) { + typesById.put(id, type); // really old stuff, should rewrite tests using this: int alt = (doc.name()+"."+doc.version()).hashCode(); - if (id == alt) { - typesById.put(id, type); - } else { - throw new IllegalArgumentException("Document type "+doc.name()+ - " wanted id "+id+" but got "+ - type.getId()+", alternative id was: "+alt); - } + log.warning("Document type "+doc.name()+ + " wanted id "+id+" but got "+ + type.getId()+", alternative id was: "+alt); } inProgress(type); configMap.remove(id); @@ -314,9 +316,255 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub private final DocumentTypeManager manager; } + + private static class ApplyNewDoctypeConfig { + + public ApplyNewDoctypeConfig(DocumentmanagerConfig config, DocumentTypeManager manager) { + this.manager = manager; + this.usev8geopositions = config.usev8geopositions(); + apply(config); + } + + Map<Integer, DataType> typesByIdx = new HashMap<>(); + + DataType addNewType(int id, DataType type) { + if (type == null) { + throw new IllegalArgumentException("Type to add for idx "+id+" cannot be null"); + } + var old = typesByIdx.put(id, type); + if (old != null) { + throw new IllegalArgumentException("Type "+type+" for idx "+id+" conflict: "+old+" present"); + } + return type; + } + + Map<Integer, Supplier<DataType>> factoryByIdx = new HashMap<>(); + + ArrayList<Integer> proxyRefs = new ArrayList<>(); + + private DataType getOrCreateType(int id) { + if (typesByIdx.containsKey(id)) { + return typesByIdx.get(id); + } + var factory = factoryByIdx.remove(id); + if (factory != null) { + DataType type = factory.get(); + return addNewType(id, type); + } + throw new IllegalArgumentException("No type or factory found for idx: "+id); + } + + void createComplexTypes() { + var toCreate = new ArrayList<Integer>(factoryByIdx.keySet()); + for (var dataTypeId : toCreate) { + var type = getOrCreateType(dataTypeId); + assert(type != null); + } + } + + class PerDocTypeData { + DocumentmanagerConfig.Doctype docTypeConfig; + + DocumentType docType = null; + + PerDocTypeData(DocumentmanagerConfig.Doctype config) { + this.docTypeConfig = config; + } + + void createSimpleTypes() { + for (var typeconf : docTypeConfig.primitivetype()) { + DataType type = manager.getDataType(typeconf.name()); + if (! (type instanceof PrimitiveDataType)) { + throw new IllegalArgumentException("Needed primitive type for idx "+typeconf.idx()+" but got: "+type); + } + addNewType(typeconf.idx(), type); + } + for (var typeconf : docTypeConfig.tensortype()) { + var type = new TensorDataType(TensorType.fromSpec(typeconf.detailedtype())); + addNewType(typeconf.idx(), type); + } + } + + void createFactories() { + for (var typeconf : docTypeConfig.arraytype()) { + factoryByIdx.put(typeconf.idx(), () -> new ArrayDataType(getOrCreateType(typeconf.elementtype()))); + } + for (var typeconf : docTypeConfig.maptype()) { + factoryByIdx.put(typeconf.idx(), () -> new MapDataType(getOrCreateType(typeconf.keytype()), + getOrCreateType(typeconf.valuetype()))); + } + for (var typeconf : docTypeConfig.wsettype()) { + factoryByIdx.put(typeconf.idx(), () -> new WeightedSetDataType(getOrCreateType(typeconf.elementtype()), + typeconf.createifnonexistent(), + typeconf.removeifzero())); + } + for (var typeconf : docTypeConfig.documentref()) { + factoryByIdx.put(typeconf.idx(), () -> ReferenceDataType.createWithInferredId(inProgressById.get(typeconf.targettype()).docType)); + } + for (var typeconf : docTypeConfig.annotationref()) { + factoryByIdx.put(typeconf.idx(), () -> new AnnotationReferenceDataType + (annTypeFromIdx(typeconf.annotationtype()))); + } + } + + void createEmptyStructs() { + String docName = docTypeConfig.name(); + for (var typeconf : docTypeConfig.structtype()) { + addNewType(typeconf.idx(), new StructDataType(typeconf.name())); + } + } + + void initializeDocType() { + Set<String> importedFields = new HashSet<>(); + for (var imported : docTypeConfig.importedfield()) { + importedFields.add(imported.name()); + } + int contentIdx = docTypeConfig.contentstruct(); + DataType contentStruct = typesByIdx.get(contentIdx); + if (! (contentStruct instanceof StructDataType)) { + throw new IllegalArgumentException("Content struct for document type "+docTypeConfig.name()+ + " should be a struct, but was: "+contentStruct); + } + if (docTypeConfig.name().equals(DataType.DOCUMENT.getName())) { + this.docType = DataType.DOCUMENT; + } else { + this.docType = new DocumentType(docTypeConfig.name(), (StructDataType)contentStruct, importedFields); + } + addNewType(docTypeConfig.idx(), docType); + } + + void createEmptyAnnotationTypes() { + for (var typeconf : docTypeConfig.annotationtype()) { + AnnotationType annType = manager.getAnnotationTypeRegistry().getType(typeconf.name()); + if (typeconf.internalid() != -1) { + if (annType == null) { + annType = new AnnotationType(typeconf.name(), typeconf.internalid()); + } else { + if (annType.getId() != typeconf.internalid()) { + throw new IllegalArgumentException("Wrong internalid for annotation type "+annType+ + " (wanted "+typeconf.internalid()+", got "+annType.getId()+")"); + } + } + } else if (annType == null) { + annType = new AnnotationType(typeconf.name()); + } + manager.getAnnotationTypeRegistry().register(annType); + // because AnnotationType is not a DataType, make a proxy + var proxy = new AnnotationReferenceDataType(annType); + proxyRefs.add(typeconf.idx()); + addNewType(typeconf.idx(), proxy); + } + } + + AnnotationType annTypeFromIdx(int idx) { + var proxy = (AnnotationReferenceDataType) typesByIdx.get(idx); + if (proxy == null) { + throw new IllegalArgumentException("Needed AnnotationType for idx "+idx+", found: "+typesByIdx.get(idx)); + } + return proxy.getAnnotationType(); + } + + void fillAnnotationTypes() { + for (var typeConf : docTypeConfig.annotationtype()) { + var annType = annTypeFromIdx(typeConf.idx()); + int pIdx = typeConf.datatype(); + if (pIdx != -1) { + DataType payload = getOrCreateType(pIdx); + annType.setDataType(payload); + } + for (var inherit : typeConf.inherits()) { + var inheritedType = annTypeFromIdx(inherit.idx()); + if (! annType.inherits(inheritedType)) { + annType.inherit(inheritedType); + } + } + } + } + void fillStructs() { + for (var structCfg : docTypeConfig.structtype()) { + int idx = structCfg.idx(); + StructDataType type = (StructDataType) typesByIdx.get(idx); + for (var parent : structCfg.inherits()) { + var parentStruct = (StructDataType) typesByIdx.get(parent.type()); + type.inherit(parentStruct); + } + for (var fieldCfg : structCfg.field()) { + if (fieldCfg.type() == idx) { + log.fine("Self-referencing struct "+structCfg.name()+" field: "+fieldCfg); + } + DataType fieldType = getOrCreateType(fieldCfg.type()); + type.addField(new Field(fieldCfg.name(), fieldCfg.internalid(), fieldType)); + } + } + } + void fillDocument() { + for (var inherit : docTypeConfig.inherits()) { + var data = inProgressById.get(inherit.idx()); + if (data == null) { + throw new IllegalArgumentException("Missing doctype for inherit idx: "+inherit.idx()); + } else { + docType.inherit(data.docType); + } + } + Map<String, Collection<String>> fieldSets = new HashMap<>(); + for (var entry : docTypeConfig.fieldsets().entrySet()) { + fieldSets.put(entry.getKey(), entry.getValue().fields()); + } + Set<String> importedFields = new HashSet<>(); + for (var imported : docTypeConfig.importedfield()) { + importedFields.add(imported.name()); + } + docType.addFieldSets(fieldSets); + } + } + + private Map<String, PerDocTypeData> inProgressByName = new HashMap<>(); + private Map<Integer, PerDocTypeData> inProgressById = new HashMap<>(); + + private void apply(DocumentmanagerConfig config) { + for (var docType : config.doctype()) { + int idx = docType.idx(); + String name = docType.name(); + var data = new PerDocTypeData(docType); + var old = inProgressById.put(idx, data); + if (old != null) { + throw new IllegalArgumentException("Multiple document types with id: "+idx); + } + old = inProgressByName.put(name, data); + if (old != null) { + throw new IllegalArgumentException("Multiple document types with name: "+name); + } + } + for (var docType : config.doctype()) { + var docTypeData = inProgressById.get(docType.idx()); + docTypeData.createEmptyStructs(); + docTypeData.initializeDocType(); + docTypeData.createEmptyAnnotationTypes(); + docTypeData.createFactories(); + docTypeData.createSimpleTypes(); + } + createComplexTypes(); + for (var docType : config.doctype()) { + var docTypeData = inProgressById.get(docType.idx()); + docTypeData.fillStructs(); + docTypeData.fillDocument(); + docTypeData.fillAnnotationTypes(); + } + for (int idx : proxyRefs) { + typesByIdx.remove(idx); + } + for (DataType type : typesByIdx.values()) { + manager.register(type); + } + } + + private final boolean usev8geopositions; + private final DocumentTypeManager manager; + } + public static DocumentTypeManager configureNewManager(DocumentmanagerConfig config) { DocumentTypeManager manager = new DocumentTypeManager(); - new Apply(config, manager); + configureNewManager(config, manager); return manager; } |