diff options
93 files changed, 840 insertions, 3035 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/Application.java b/config-model/src/main/java/com/yahoo/schema/Application.java index b147388747e..71c0e563c77 100644 --- a/config-model/src/main/java/com/yahoo/schema/Application.java +++ b/config-model/src/main/java/com/yahoo/schema/Application.java @@ -54,8 +54,6 @@ public class Application { if (validate) schemas.forEach(schema -> schema.validate(logger)); - new TemporarySDTypeResolver(schemas, logger).process(); - List<SDDocumentType> sdocs = new ArrayList<>(); sdocs.add(SDDocumentType.VESPA_DOCUMENT); for (Schema schema : schemas) { @@ -64,9 +62,6 @@ public class Application { } } - var orderer = new SDDocumentTypeOrderer(sdocs, logger); - orderer.process(); - var resolver = new DocumentReferenceResolver(schemas); sdocs.forEach(resolver::resolveReferences); sdocs.forEach(resolver::resolveInheritedReferences); diff --git a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java index 29482431aa0..d98709569b1 100644 --- a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java @@ -132,8 +132,11 @@ public class ApplicationBuilder { this.deployLogger = deployLogger; this.properties = properties; this.documentsOnly = documentsOnly; - for (NamedReader reader : applicationPackage.getSchemas()) + var list = new ArrayList<>(applicationPackage.getSchemas()); + list.sort((a, b) -> a.getName().compareTo(b.getName())); + for (NamedReader reader : list) { addSchema(reader); + } } /** @@ -407,8 +410,13 @@ public class ApplicationBuilder { properties, rankProfileRegistry, queryProfileRegistry); - for (var i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { - builder.addSchemaFile(i.next().toString()); + + var fnli = Files.list(new File(dir).toPath()) + .map(p -> p.toString()) + .filter(fn -> fn.endsWith(".sd")) + .sorted(); + for (var i = fnli.iterator(); i.hasNext(); ) { + builder.addSchemaFile(i.next()); } builder.build(true); return builder; diff --git a/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java deleted file mode 100644 index 64bca9367d8..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.*; -import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.schema.document.SDDocumentType; -import com.yahoo.schema.document.TemporarySDDocumentType; - -import java.util.*; -import java.util.logging.Level; - -/** - * @author Einar M R Rosenvinge - */ -public class SDDocumentTypeOrderer { - - private final Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>(); - private final Set<Object> seenTypes = Collections.newSetFromMap(new IdentityHashMap<>()); - List<SDDocumentType> processingOrder = new LinkedList<>(); - private final DeployLogger deployLogger; - - public SDDocumentTypeOrderer(List<SDDocumentType> sdTypes, DeployLogger deployLogger) { - this.deployLogger = deployLogger; - for (SDDocumentType type : sdTypes) { - createdSDTypes.put(type.getDocumentName(), type); - } - } - - List<SDDocumentType> getOrdered() { return processingOrder; } - - public void process() { - for (SDDocumentType type : createdSDTypes.values()) { - process(type, type); - } - } - - private void process(SDDocumentType docOrStruct, SDDocumentType owningDocument) { - resolveAndProcessInheritedTemporaryTypes(docOrStruct, owningDocument); - if (seenTypes.contains(docOrStruct)) { - return; - } - seenTypes.add(docOrStruct); - for (Field field : docOrStruct.fieldSet()) { - var type = field.getDataType(); - String typeName = type.getName(); - if (!seenTypes.contains(type)) { - seenTypes.add(type); - //we haven't seen this before, do it - visit(type, owningDocument); - } - } - processingOrder.add(docOrStruct); - } - - private void resolveAndProcessInheritedTemporaryTypes(SDDocumentType type, SDDocumentType owningDocument) { - List<DataTypeName> toReplace = new ArrayList<>(); - for (SDDocumentType sdoc : type.getInheritedTypes()) { - if (sdoc instanceof TemporarySDDocumentType) { - toReplace.add(sdoc.getDocumentName()); - } - } - for (DataTypeName name : toReplace) { - SDDocumentType inherited; - if (type.isStruct()) { - inherited = owningDocument.allTypes().get(new NewDocumentType.Name(name.getName())); - if (inherited == null) throw new IllegalArgumentException("Struct '" + name + "' not found in " + owningDocument); - process(inherited, owningDocument); - } - else { - inherited = createdSDTypes.get(name); - if (inherited == null) { - throw new IllegalArgumentException("document " + type.getName() + - " inherits from unavailable document " + name); - } - process(inherited, inherited); - } - type.inherit(inherited); - } - } - - private SDDocumentType find(String name) { - SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name)); - if (sdDocType != null) { - return sdDocType; - } - for(SDDocumentType sdoc : createdSDTypes.values()) { - for (SDDocumentType stype : sdoc.getTypes()) { - if (stype.getName().equals(name)) { - return stype; - } - } - } - return null; - } - - private void visit(DataType type, SDDocumentType owningDocument) { - if (type instanceof StructuredDataType) { - StructuredDataType structType = (StructuredDataType) type; - SDDocumentType sdDocType = owningDocument.getType(structType.getName()); - if (sdDocType == null) { - sdDocType = find(structType.getName()); - } - if (sdDocType == null) { - throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'"); - } - process(sdDocType, owningDocument); - return; - } - - if (type instanceof MapDataType) { - MapDataType mType = (MapDataType) type; - visit(mType.getValueType(), owningDocument); - visit(mType.getKeyType(), owningDocument); - } else if (type instanceof WeightedSetDataType) { - WeightedSetDataType wType = (WeightedSetDataType) type; - visit(wType.getNestedType(), owningDocument); - } else if (type instanceof CollectionDataType) { - CollectionDataType cType = (CollectionDataType) type; - visit(cType.getNestedType(), owningDocument); - } else if (type instanceof AnnotationReferenceDataType) { - //do nothing - } else if (type instanceof PrimitiveDataType) { - //do nothing - } else if (type instanceof TensorDataType) { - //do nothing - } else if (type instanceof NewDocumentReferenceDataType) { - //do nothing - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unknown type : " + type); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java deleted file mode 100644 index b1ce6f5eb4f..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.schema.document.SDDocumentType; -import com.yahoo.schema.document.TemporarySDDocumentType; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.logging.Level; - -/** - * @author arnej - */ -public class TemporarySDTypeResolver { - - private final DeployLogger deployLogger; - private final Collection<Schema> toProcess; - private final List<SDDocumentType> docTypes = new LinkedList<>(); - - public TemporarySDTypeResolver(Collection<Schema> schemas, DeployLogger deployLogger) { - this.deployLogger = deployLogger; - this.toProcess = schemas; - } - - private SDDocumentType findDocType(String name) { - assert(name != null); - for (var doc : docTypes) { - if (doc.getName().equals(name)) { - return doc; - } - } - deployLogger.logApplicationPackage(Level.WARNING, "No document type in application matching name: "+name); - return null; - } - - public void process() { - docTypes.add(SDDocumentType.VESPA_DOCUMENT); - for (Schema schema : toProcess) { - if (schema.hasDocument()) { - docTypes.add(schema.getDocument()); - } - } - // first, fix inheritance - for (SDDocumentType doc : docTypes) { - for (SDDocumentType inherited : doc.getInheritedTypes()) { - if (inherited instanceof TemporarySDDocumentType) { - var actual = findDocType(inherited.getName()); - if (actual != null) { - doc.inherit(actual); - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unresolved inherit '"+inherited.getName() +"' for document "+doc.getName()); - } - } - } - } - // next, check owned types (structs only?) - for (SDDocumentType doc : docTypes) { - for (SDDocumentType owned : doc.getTypes()) { - if (owned instanceof TemporarySDDocumentType) { - deployLogger.logApplicationPackage(Level.WARNING, "Schema '"+doc.getName()+"' owned type '"+owned.getName()+"' is temporary, should not happen"); - continue; - } - for (SDDocumentType inherited : owned.getInheritedTypes()) { - if (inherited instanceof TemporarySDDocumentType) { - var actual = doc.getType(inherited.getName()); - if (actual != null) { - owned.inherit(actual); - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unresolved inherit '"+inherited.getName() +"' for type '"+owned.getName()+"' in document "+doc.getName()); - } - } - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java b/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java index 3bab808beff..d08cc472f82 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java @@ -33,10 +33,6 @@ public class SearchOrderer { * @return a new list containing the same search instances in the right order */ public List<Schema> order(List<Schema> unordered) { - // Description above state that the original order should be preserved, except for the dependency constraint. - // Yet we botch that guarantee by sorting the list... - unordered.sort(Comparator.comparing(Schema::getName)); - // No, this is not a fast algorithm... indexOnDocumentName(unordered); List<Schema> ordered = new ArrayList<>(unordered.size()); diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java index d19b62ce555..919a6023151 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java @@ -167,16 +167,11 @@ public class SDDocumentType implements Cloneable { public DataTypeName getDocumentName() { return docType.getDataTypeName(); } public DocumentType getDocumentType() { return docType; } - public void inherit(DataTypeName name) { - inherit(new TemporarySDDocumentType(name)); - } - public void inherit(SDDocumentType type) { if (type == null) return; if (type.getName().equals(this.getName())) throw new IllegalArgumentException("Document type '" + getName() + "' cannot inherit itself"); - if ( ! inheritedTypes.containsKey(type.getDocumentName()) || - (inheritedTypes.get(type.getDocumentName()) instanceof TemporarySDDocumentType)) { + if (! inheritedTypes.containsKey(type.getDocumentName())) { inheritedTypes.put(type.getDocumentName(), type); } } diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java deleted file mode 100644 index 3194a14a143..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.document; - -import com.yahoo.document.DataTypeName; - -/** - * @author baldersheim - */ -public class TemporarySDDocumentType extends SDDocumentType { - public TemporarySDDocumentType(DataTypeName name) { - super(name); - } -} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 3008dac4f29..443dfb1602d 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import java.util.Locale; +import java.util.Map; /** * Helper for converting ParsedField etc to SDField with settings @@ -29,9 +30,11 @@ import java.util.Locale; public class ConvertParsedFields { private final TypeResolver context; + private final Map<String, SDDocumentType> structProxies; - ConvertParsedFields(TypeResolver context) { + ConvertParsedFields(TypeResolver context, Map<String, SDDocumentType> structProxies) { this.context = context; + this.structProxies = structProxies; } static void convertMatchSettings(SDField field, ParsedMatchSettings parsed) { @@ -309,10 +312,11 @@ public class ConvertParsedFields { structProxy.setFieldId(field, parsedField.idOverride()); } } - for (String inherit : parsed.getInherited()) { - structProxy.inherit(new DataTypeName(inherit)); + for (var inherit: parsed.getResolvedInherits()) { + structProxy.inherit(structProxies.get(inherit.getFullName())); } structProxy.setStruct(context.resolveStruct(parsed)); + structProxies.put(parsed.getFullName(), structProxy); return structProxy; } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index f3289621ce1..0abcc9e890a 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -80,6 +80,7 @@ public class ConvertParsedSchemas { } private final Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap<>(); + private final Map<String, SDDocumentType> convertedStructs = new LinkedHashMap<>(); public List<Schema> convertToSchemas() { typeConverter.convert(false); @@ -187,7 +188,7 @@ public class ConvertParsedSchemas { } parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); var typeContext = typeConverter.makeContext(parsed.getDocument()); - var fieldConverter = new ConvertParsedFields(typeContext); + var fieldConverter = new ConvertParsedFields(typeContext, convertedStructs); convertDocument(schema, parsed.getDocument(), fieldConverter); for (var field : parsed.getFields()) { fieldConverter.convertExtraField(schema, field); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java index abe14b3689f..02d10bcb487 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java @@ -27,8 +27,13 @@ public class ParsedStruct extends ParsedBlock { List<ParsedField> getFields() { return List.copyOf(fields.values()); } List<String> getInherited() { return List.copyOf(inherited); } - ParsedDocument getOwnerDoc() { return ownedBy; } - String getOwnerName() { return ownedBy.name(); } + ParsedDocument getOwnerDoc() { + verifyThat(ownedBy != null, "not owned by any document"); + return ownedBy; + } + String getOwnerName() { return getOwnerDoc().name(); } + String getFullName() { return name() + " @ " + getOwnerName(); } + List<ParsedStruct> getResolvedInherits() { assert(inherited.size() == resolvedInherits.size()); return List.copyOf(resolvedInherits); diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg index eaa7e2c5131..a4b37113ef1 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -39,48 +39,48 @@ doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "person_ref" -doctype[1].importedfield[0].name "my_cool_field" -doctype[1].importedfield[1].name "my_swag_field" -doctype[1].importedfield[2].name "my_name" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10020 +doctype[1].fieldsets{[document]}.fields[0] "cool_field" +doctype[1].fieldsets{[document]}.fields[1] "swag_field" doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "person_ref" -doctype[1].structtype[0].field[1].internalid 100779805 -doctype[1].structtype[0].field[1].type 10019 -doctype[2].name "campaign" -doctype[2].idx 10018 +doctype[1].structtype[0].name "campaign.header" +doctype[1].structtype[0].field[0].name "cool_field" +doctype[1].structtype[0].field[0].internalid 1588702436 +doctype[1].structtype[0].field[0].type 10012 +doctype[1].structtype[0].field[1].name "swag_field" +doctype[1].structtype[0].field[1].internalid 1691224741 +doctype[1].structtype[0].field[1].type 10008 +doctype[2].name "person" +doctype[2].idx 10017 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10021 -doctype[2].fieldsets{[document]}.fields[0] "cool_field" -doctype[2].fieldsets{[document]}.fields[1] "swag_field" -doctype[2].structtype[0].idx 10021 -doctype[2].structtype[0].name "campaign.header" -doctype[2].structtype[0].field[0].name "cool_field" -doctype[2].structtype[0].field[0].internalid 1588702436 +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "name" +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "person.header" +doctype[2].structtype[0].field[0].name "name" +doctype[2].structtype[0].field[0].internalid 1160796772 doctype[2].structtype[0].field[0].type 10012 -doctype[2].structtype[0].field[1].name "swag_field" -doctype[2].structtype[0].field[1].internalid 1691224741 -doctype[2].structtype[0].field[1].type 10008 -doctype[3].name "person" -doctype[3].idx 10020 +doctype[3].name "ad" +doctype[3].idx 10019 doctype[3].inherits[0].idx 10000 -doctype[3].contentstruct 10022 -doctype[3].fieldsets{[document]}.fields[0] "name" -doctype[3].structtype[0].idx 10022 -doctype[3].structtype[0].name "person.header" -doctype[3].structtype[0].field[0].name "name" -doctype[3].structtype[0].field[0].internalid 1160796772 -doctype[3].structtype[0].field[0].type 10012 +doctype[3].contentstruct 10020 +doctype[3].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[3].fieldsets{[document]}.fields[1] "person_ref" +doctype[3].importedfield[0].name "my_cool_field" +doctype[3].importedfield[1].name "my_swag_field" +doctype[3].importedfield[2].name "my_name" +doctype[3].documentref[0].idx 10021 +doctype[3].documentref[0].targettype 10015 +doctype[3].documentref[1].idx 10022 +doctype[3].documentref[1].targettype 10017 +doctype[3].structtype[0].idx 10020 +doctype[3].structtype[0].name "ad.header" +doctype[3].structtype[0].field[0].name "campaign_ref" +doctype[3].structtype[0].field[0].internalid 23963250 +doctype[3].structtype[0].field[0].type 10021 +doctype[3].structtype[0].field[1].name "person_ref" +doctype[3].structtype[0].field[1].internalid 100779805 +doctype[3].structtype[0].field[1].type 10022 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg index 48348348989..f6d4269abcd 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg @@ -39,33 +39,33 @@ doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "person_ref" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10020 doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "person_ref" -doctype[1].structtype[0].field[1].internalid 100779805 -doctype[1].structtype[0].field[1].type 10019 -doctype[2].name "campaign" -doctype[2].idx 10018 +doctype[1].structtype[0].name "campaign.header" +doctype[2].name "person" +doctype[2].idx 10017 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10021 -doctype[2].structtype[0].idx 10021 -doctype[2].structtype[0].name "campaign.header" -doctype[3].name "person" -doctype[3].idx 10020 +doctype[2].contentstruct 10018 +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "person.header" +doctype[3].name "ad" +doctype[3].idx 10019 doctype[3].inherits[0].idx 10000 -doctype[3].contentstruct 10022 -doctype[3].structtype[0].idx 10022 -doctype[3].structtype[0].name "person.header" +doctype[3].contentstruct 10020 +doctype[3].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[3].fieldsets{[document]}.fields[1] "person_ref" +doctype[3].documentref[0].idx 10021 +doctype[3].documentref[0].targettype 10015 +doctype[3].documentref[1].idx 10022 +doctype[3].documentref[1].targettype 10017 +doctype[3].structtype[0].idx 10020 +doctype[3].structtype[0].name "ad.header" +doctype[3].structtype[0].field[0].name "campaign_ref" +doctype[3].structtype[0].field[0].internalid 23963250 +doctype[3].structtype[0].field[0].type 10021 +doctype[3].structtype[0].field[1].name "person_ref" +doctype[3].structtype[0].field[1].internalid 100779805 +doctype[3].structtype[0].field[1].type 10022 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index ef1bb4c5ad4..31f514b16cd 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -39,25 +39,25 @@ doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "other_campaign_ref" -doctype[1].structtype[0].field[1].internalid 874751172 -doctype[1].structtype[0].field[1].type 10017 -doctype[2].name "campaign" -doctype[2].idx 10018 +doctype[1].structtype[0].name "campaign.header" +doctype[2].name "ad" +doctype[2].idx 10017 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10019 -doctype[2].structtype[0].idx 10019 -doctype[2].structtype[0].name "campaign.header" +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[2].fieldsets{[document]}.fields[1] "other_campaign_ref" +doctype[2].documentref[0].idx 10019 +doctype[2].documentref[0].targettype 10015 +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "ad.header" +doctype[2].structtype[0].field[0].name "campaign_ref" +doctype[2].structtype[0].field[0].internalid 23963250 +doctype[2].structtype[0].field[0].type 10019 +doctype[2].structtype[0].field[1].name "other_campaign_ref" +doctype[2].structtype[0].field[1].internalid 874751172 +doctype[2].structtype[0].field[1].type 10019 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg index 1b5817e6f39..1582c6572da 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -43,56 +43,56 @@ doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 doctype[0].structtype[1].internalid 1381038251 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 -doctype[1].internalid 2987301 +doctype[1].internalid -1318255918 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "person_ref" -doctype[1].importedfield[0].name "my_cool_field" -doctype[1].importedfield[1].name "my_swag_field" -doctype[1].importedfield[2].name "my_name" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[0].internalid 595216861 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10020 -doctype[1].documentref[1].internalid 542332920 +doctype[1].fieldsets{[document]}.fields[0] "cool_field" +doctype[1].fieldsets{[document]}.fields[1] "swag_field" doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "person_ref" -doctype[1].structtype[0].field[1].internalid 100779805 -doctype[1].structtype[0].field[1].type 10019 -doctype[1].structtype[0].internalid 959075962 -doctype[2].name "campaign" -doctype[2].idx 10018 -doctype[2].internalid -1318255918 +doctype[1].structtype[0].name "campaign.header" +doctype[1].structtype[0].field[0].name "cool_field" +doctype[1].structtype[0].field[0].internalid 1588702436 +doctype[1].structtype[0].field[0].type 10012 +doctype[1].structtype[0].field[1].name "swag_field" +doctype[1].structtype[0].field[1].internalid 1691224741 +doctype[1].structtype[0].field[1].type 10008 +doctype[1].structtype[0].internalid -2041471955 +doctype[2].name "person" +doctype[2].idx 10017 +doctype[2].internalid 443162583 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10021 -doctype[2].fieldsets{[document]}.fields[0] "cool_field" -doctype[2].fieldsets{[document]}.fields[1] "swag_field" -doctype[2].structtype[0].idx 10021 -doctype[2].structtype[0].name "campaign.header" -doctype[2].structtype[0].field[0].name "cool_field" -doctype[2].structtype[0].field[0].internalid 1588702436 +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "name" +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "person.header" +doctype[2].structtype[0].field[0].name "name" +doctype[2].structtype[0].field[0].internalid 1160796772 doctype[2].structtype[0].field[0].type 10012 -doctype[2].structtype[0].field[1].name "swag_field" -doctype[2].structtype[0].field[1].internalid 1691224741 -doctype[2].structtype[0].field[1].type 10008 -doctype[2].structtype[0].internalid -2041471955 -doctype[3].name "person" -doctype[3].idx 10020 -doctype[3].internalid 443162583 +doctype[2].structtype[0].internalid 3129224 +doctype[3].name "ad" +doctype[3].idx 10019 +doctype[3].internalid 2987301 doctype[3].inherits[0].idx 10000 -doctype[3].contentstruct 10022 -doctype[3].fieldsets{[document]}.fields[0] "name" -doctype[3].structtype[0].idx 10022 -doctype[3].structtype[0].name "person.header" -doctype[3].structtype[0].field[0].name "name" -doctype[3].structtype[0].field[0].internalid 1160796772 -doctype[3].structtype[0].field[0].type 10012 -doctype[3].structtype[0].internalid 3129224 +doctype[3].contentstruct 10020 +doctype[3].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[3].fieldsets{[document]}.fields[1] "person_ref" +doctype[3].importedfield[0].name "my_cool_field" +doctype[3].importedfield[1].name "my_swag_field" +doctype[3].importedfield[2].name "my_name" +doctype[3].documentref[0].idx 10021 +doctype[3].documentref[0].targettype 10015 +doctype[3].documentref[0].internalid 595216861 +doctype[3].documentref[1].idx 10022 +doctype[3].documentref[1].targettype 10017 +doctype[3].documentref[1].internalid 542332920 +doctype[3].structtype[0].idx 10020 +doctype[3].structtype[0].name "ad.header" +doctype[3].structtype[0].field[0].name "campaign_ref" +doctype[3].structtype[0].field[0].internalid 23963250 +doctype[3].structtype[0].field[0].type 10021 +doctype[3].structtype[0].field[1].name "person_ref" +doctype[3].structtype[0].field[1].internalid 100779805 +doctype[3].structtype[0].field[1].type 10022 +doctype[3].structtype[0].internalid 959075962 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg index 1c5d4d41819..4222229cdd3 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg @@ -43,41 +43,41 @@ doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 doctype[0].structtype[1].internalid 1381038251 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 -doctype[1].internalid 2987301 +doctype[1].internalid -1318255918 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "person_ref" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[0].internalid 595216861 -doctype[1].documentref[1].idx 10019 -doctype[1].documentref[1].targettype 10020 -doctype[1].documentref[1].internalid 542332920 doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "person_ref" -doctype[1].structtype[0].field[1].internalid 100779805 -doctype[1].structtype[0].field[1].type 10019 -doctype[1].structtype[0].internalid 959075962 -doctype[2].name "campaign" -doctype[2].idx 10018 -doctype[2].internalid -1318255918 +doctype[1].structtype[0].name "campaign.header" +doctype[1].structtype[0].internalid -2041471955 +doctype[2].name "person" +doctype[2].idx 10017 +doctype[2].internalid 443162583 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10021 -doctype[2].structtype[0].idx 10021 -doctype[2].structtype[0].name "campaign.header" -doctype[2].structtype[0].internalid -2041471955 -doctype[3].name "person" -doctype[3].idx 10020 -doctype[3].internalid 443162583 +doctype[2].contentstruct 10018 +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "person.header" +doctype[2].structtype[0].internalid 3129224 +doctype[3].name "ad" +doctype[3].idx 10019 +doctype[3].internalid 2987301 doctype[3].inherits[0].idx 10000 -doctype[3].contentstruct 10022 -doctype[3].structtype[0].idx 10022 -doctype[3].structtype[0].name "person.header" -doctype[3].structtype[0].internalid 3129224 +doctype[3].contentstruct 10020 +doctype[3].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[3].fieldsets{[document]}.fields[1] "person_ref" +doctype[3].documentref[0].idx 10021 +doctype[3].documentref[0].targettype 10015 +doctype[3].documentref[0].internalid 595216861 +doctype[3].documentref[1].idx 10022 +doctype[3].documentref[1].targettype 10017 +doctype[3].documentref[1].internalid 542332920 +doctype[3].structtype[0].idx 10020 +doctype[3].structtype[0].name "ad.header" +doctype[3].structtype[0].field[0].name "campaign_ref" +doctype[3].structtype[0].field[0].internalid 23963250 +doctype[3].structtype[0].field[0].type 10021 +doctype[3].structtype[0].field[1].name "person_ref" +doctype[3].structtype[0].field[1].internalid 100779805 +doctype[3].structtype[0].field[1].type 10022 +doctype[3].structtype[0].internalid 959075962 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg index 2f178c55bfd..0d3802bcbe3 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg @@ -43,30 +43,30 @@ doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 doctype[0].structtype[1].internalid 1381038251 -doctype[1].name "ad" +doctype[1].name "campaign" doctype[1].idx 10015 -doctype[1].internalid 2987301 +doctype[1].internalid -1318255918 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "campaign_ref" -doctype[1].fieldsets{[document]}.fields[1] "other_campaign_ref" -doctype[1].documentref[0].idx 10017 -doctype[1].documentref[0].targettype 10018 -doctype[1].documentref[0].internalid 595216861 doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "ad.header" -doctype[1].structtype[0].field[0].name "campaign_ref" -doctype[1].structtype[0].field[0].internalid 23963250 -doctype[1].structtype[0].field[0].type 10017 -doctype[1].structtype[0].field[1].name "other_campaign_ref" -doctype[1].structtype[0].field[1].internalid 874751172 -doctype[1].structtype[0].field[1].type 10017 -doctype[1].structtype[0].internalid 959075962 -doctype[2].name "campaign" -doctype[2].idx 10018 -doctype[2].internalid -1318255918 +doctype[1].structtype[0].name "campaign.header" +doctype[1].structtype[0].internalid -2041471955 +doctype[2].name "ad" +doctype[2].idx 10017 +doctype[2].internalid 2987301 doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10019 -doctype[2].structtype[0].idx 10019 -doctype[2].structtype[0].name "campaign.header" -doctype[2].structtype[0].internalid -2041471955 +doctype[2].contentstruct 10018 +doctype[2].fieldsets{[document]}.fields[0] "campaign_ref" +doctype[2].fieldsets{[document]}.fields[1] "other_campaign_ref" +doctype[2].documentref[0].idx 10019 +doctype[2].documentref[0].targettype 10015 +doctype[2].documentref[0].internalid 595216861 +doctype[2].structtype[0].idx 10018 +doctype[2].structtype[0].name "ad.header" +doctype[2].structtype[0].field[0].name "campaign_ref" +doctype[2].structtype[0].field[0].internalid 23963250 +doctype[2].structtype[0].field[0].type 10019 +doctype[2].structtype[0].field[1].name "other_campaign_ref" +doctype[2].structtype[0].field[1].internalid 874751172 +doctype[2].structtype[0].field[1].type 10019 +doctype[2].structtype[0].internalid 959075962 diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg index 7a67640adfe..b15734169a5 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg @@ -43,57 +43,57 @@ doctype[].structtype[].field[].name "y" doctype[].structtype[].field[].internalid 900009410 doctype[].structtype[].field[].type 10007 doctype[].structtype[].internalid 1381038251 -doctype[].name "child_a" +doctype[].name "parent" doctype[].idx 10015 -doctype[].internalid -94853056 +doctype[].internalid 1175161836 doctype[].inherits[].idx 10000 doctype[].contentstruct 10016 +doctype[].fieldsets{[document]}.fields[] "int_field" +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "parent.header" +doctype[].structtype[].field[].name "int_field" +doctype[].structtype[].field[].internalid 2128822283 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 836075987 +doctype[].name "child_a" +doctype[].idx 10017 +doctype[].internalid -94853056 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10018 doctype[].fieldsets{[document]}.fields[] "ref_from_a" -doctype[].documentref[].idx 10017 -doctype[].documentref[].targettype 10018 +doctype[].documentref[].idx 10019 +doctype[].documentref[].targettype 10015 doctype[].documentref[].internalid 427398467 -doctype[].structtype[].idx 10016 +doctype[].structtype[].idx 10018 doctype[].structtype[].name "child_a.header" doctype[].structtype[].field[].name "ref_from_a" doctype[].structtype[].field[].internalid 300427062 -doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].type 10019 doctype[].structtype[].internalid 867409663 doctype[].name "child_b" -doctype[].idx 10019 +doctype[].idx 10020 doctype[].internalid -94852095 doctype[].inherits[].idx 10000 -doctype[].inherits[].idx 10015 -doctype[].contentstruct 10020 +doctype[].inherits[].idx 10017 +doctype[].contentstruct 10021 doctype[].fieldsets{[document]}.fields[] "ref_from_a" doctype[].fieldsets{[document]}.fields[] "ref_from_b" -doctype[].structtype[].idx 10020 +doctype[].structtype[].idx 10021 doctype[].structtype[].name "child_b.header" doctype[].structtype[].field[].name "ref_from_b" doctype[].structtype[].field[].internalid 185778735 -doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].type 10019 doctype[].structtype[].internalid 670896158 doctype[].name "child_c" -doctype[].idx 10021 +doctype[].idx 10022 doctype[].internalid -94851134 doctype[].inherits[].idx 10000 -doctype[].inherits[].idx 10019 -doctype[].contentstruct 10022 +doctype[].inherits[].idx 10020 +doctype[].contentstruct 10023 doctype[].fieldsets{[document]}.fields[] "ref_from_a" doctype[].fieldsets{[document]}.fields[] "ref_from_b" doctype[].importedfield[].name "from_a_int_field" doctype[].importedfield[].name "from_b_int_field" -doctype[].structtype[].idx 10022 +doctype[].structtype[].idx 10023 doctype[].structtype[].name "child_c.header" doctype[].structtype[].internalid 474382653 -doctype[].name "parent" -doctype[].idx 10018 -doctype[].internalid 1175161836 -doctype[].inherits[].idx 10000 -doctype[].contentstruct 10023 -doctype[].fieldsets{[document]}.fields[] "int_field" -doctype[].structtype[].idx 10023 -doctype[].structtype[].name "parent.header" -doctype[].structtype[].field[].name "int_field" -doctype[].structtype[].field[].internalid 2128822283 -doctype[].structtype[].field[].type 10007 -doctype[].structtype[].internalid 836075987 diff --git a/config-model/src/test/derived/reference_from_several/documentmanager.cfg b/config-model/src/test/derived/reference_from_several/documentmanager.cfg index 0418b05d9d6..6ac1aab0baf 100644 --- a/config-model/src/test/derived/reference_from_several/documentmanager.cfg +++ b/config-model/src/test/derived/reference_from_several/documentmanager.cfg @@ -39,45 +39,45 @@ doctype[].structtype[].field[].type 10007 doctype[].structtype[].field[].name "y" doctype[].structtype[].field[].internalid 900009410 doctype[].structtype[].field[].type 10007 -doctype[].name "bar" +doctype[].name "parent" doctype[].idx 10015 doctype[].inherits[].idx 10000 doctype[].contentstruct 10016 -doctype[].fieldsets{[document]}.fields[] "bartitle" -doctype[].fieldsets{[document]}.fields[] "bpref" -doctype[].importedfield[].name "barsximp" -doctype[].documentref[].idx 10017 -doctype[].documentref[].targettype 10018 +doctype[].fieldsets{[document]}.fields[] "x" doctype[].structtype[].idx 10016 -doctype[].structtype[].name "bar.header" -doctype[].structtype[].field[].name "bpref" -doctype[].structtype[].field[].internalid 1709838545 -doctype[].structtype[].field[].type 10017 -doctype[].structtype[].field[].name "bartitle" -doctype[].structtype[].field[].internalid 1554393914 -doctype[].structtype[].field[].type 10012 +doctype[].structtype[].name "parent.header" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 doctype[].name "foo" -doctype[].idx 10019 +doctype[].idx 10017 doctype[].inherits[].idx 10000 -doctype[].contentstruct 10020 +doctype[].contentstruct 10018 doctype[].fieldsets{[document]}.fields[] "foo" doctype[].fieldsets{[document]}.fields[] "myref" doctype[].importedfield[].name "myx" -doctype[].structtype[].idx 10020 +doctype[].documentref[].idx 10019 +doctype[].documentref[].targettype 10015 +doctype[].structtype[].idx 10018 doctype[].structtype[].name "foo.header" doctype[].structtype[].field[].name "myref" doctype[].structtype[].field[].internalid 598565475 -doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].type 10019 doctype[].structtype[].field[].name "foo" doctype[].structtype[].field[].internalid 846279091 doctype[].structtype[].field[].type 10012 -doctype[].name "parent" -doctype[].idx 10018 +doctype[].name "bar" +doctype[].idx 10020 doctype[].inherits[].idx 10000 doctype[].contentstruct 10021 -doctype[].fieldsets{[document]}.fields[] "x" +doctype[].fieldsets{[document]}.fields[] "bartitle" +doctype[].fieldsets{[document]}.fields[] "bpref" +doctype[].importedfield[].name "barsximp" doctype[].structtype[].idx 10021 -doctype[].structtype[].name "parent.header" -doctype[].structtype[].field[].name "x" -doctype[].structtype[].field[].internalid 914677694 -doctype[].structtype[].field[].type 10007 +doctype[].structtype[].name "bar.header" +doctype[].structtype[].field[].name "bpref" +doctype[].structtype[].field[].internalid 1709838545 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "bartitle" +doctype[].structtype[].field[].internalid 1554393914 +doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index ea56800ea1c..b59b3206a2d 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -39,31 +39,31 @@ doctype[0].structtype[1].field[0].type 10007 doctype[0].structtype[1].field[1].name "y" doctype[0].structtype[1].field[1].internalid 900009410 doctype[0].structtype[1].field[1].type 10007 -doctype[1].name "book" +doctype[1].name "music" doctype[1].idx 10015 doctype[1].inherits[0].idx 10000 doctype[1].contentstruct 10016 -doctype[1].fieldsets{[document]}.fields[0] "soundtrack" +doctype[1].fieldsets{[document]}.fields[0] "intfield" +doctype[1].fieldsets{[document]}.fields[1] "longfield" +doctype[1].fieldsets{[document]}.fields[2] "stringfield" doctype[1].structtype[0].idx 10016 -doctype[1].structtype[0].name "book.header" -doctype[1].structtype[0].field[0].name "soundtrack" -doctype[1].structtype[0].field[0].internalid 1258961213 -doctype[1].structtype[0].field[0].type 10017 -doctype[2].name "music" +doctype[1].structtype[0].name "music.header" +doctype[1].structtype[0].field[0].name "intfield" +doctype[1].structtype[0].field[0].internalid 435380425 +doctype[1].structtype[0].field[0].type 10007 +doctype[1].structtype[0].field[1].name "stringfield" +doctype[1].structtype[0].field[1].internalid 1182460484 +doctype[1].structtype[0].field[1].type 10012 +doctype[1].structtype[0].field[2].name "longfield" +doctype[1].structtype[0].field[2].internalid 1589309697 +doctype[1].structtype[0].field[2].type 10008 +doctype[2].name "book" doctype[2].idx 10017 doctype[2].inherits[0].idx 10000 doctype[2].contentstruct 10018 -doctype[2].fieldsets{[document]}.fields[0] "intfield" -doctype[2].fieldsets{[document]}.fields[1] "longfield" -doctype[2].fieldsets{[document]}.fields[2] "stringfield" +doctype[2].fieldsets{[document]}.fields[0] "soundtrack" doctype[2].structtype[0].idx 10018 -doctype[2].structtype[0].name "music.header" -doctype[2].structtype[0].field[0].name "intfield" -doctype[2].structtype[0].field[0].internalid 435380425 -doctype[2].structtype[0].field[0].type 10007 -doctype[2].structtype[0].field[1].name "stringfield" -doctype[2].structtype[0].field[1].internalid 1182460484 -doctype[2].structtype[0].field[1].type 10012 -doctype[2].structtype[0].field[2].name "longfield" -doctype[2].structtype[0].field[2].internalid 1589309697 -doctype[2].structtype[0].field[2].type 10008 +doctype[2].structtype[0].name "book.header" +doctype[2].structtype[0].field[0].name "soundtrack" +doctype[2].structtype[0].field[0].internalid 1258961213 +doctype[2].structtype[0].field[0].type 10015 diff --git a/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java deleted file mode 100755 index 45780d39021..00000000000 --- a/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.document.DataTypeName; -import com.yahoo.schema.document.SDDocumentType; -import com.yahoo.schema.document.SDField; -import com.yahoo.schema.document.TemporarySDDocumentType; -import com.yahoo.schema.document.TemporarySDField; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author Einar M R Rosenvinge - */ -public class SDDocumentTypeOrdererTestCase { - - @Test - public void testOrder() { - List<SDDocumentType> types = new ArrayList<>(); - - SDDocumentType a = new SDDocumentType("a"); - SDDocumentType b = new SDDocumentType("b"); - SDDocumentType c = new SDDocumentType("c"); - SDDocumentType d = new SDDocumentType("d"); - SDDocumentType e = new SDDocumentType("e"); - SDDocumentType f = new SDDocumentType("f"); - SDDocumentType g = new SDDocumentType("g"); - b.inherit(new TemporarySDDocumentType(new DataTypeName("a"))); - c.inherit(new TemporarySDDocumentType(new DataTypeName("b"))); - d.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); - g.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); - g.inherit(new TemporarySDDocumentType(new DataTypeName("c"))); - - SDField aFieldTypeB = new TemporarySDField(a, "atypeb", DataType.STRING); - a.addField(aFieldTypeB); - - SDField bFieldTypeC = new TemporarySDField(b, "btypec", DataType.STRING); - b.addField(bFieldTypeC); - - SDField cFieldTypeG = new TemporarySDField(c, "ctypeg", DataType.STRING); - c.addField(cFieldTypeG); - - SDField gFieldTypeF = new TemporarySDField(g, "gtypef", DataType.STRING); - g.addField(gFieldTypeF); - - SDField fFieldTypeC = new TemporarySDField(f, "ftypec", DataType.STRING); - f.addField(fFieldTypeC); - - SDField dFieldTypeE = new TemporarySDField(d, "dtypee", DataType.STRING); - d.addField(dFieldTypeE); - - types.add(a); - types.add(b); - types.add(c); - types.add(d); - types.add(e); - types.add(f); - types.add(g); - - SDDocumentTypeOrderer app = new SDDocumentTypeOrderer(types, new BaseDeployLogger()); - app.process(); - assertEquals(7, app.processingOrder.size()); - assertEquals(a, app.processingOrder.get(0)); - assertEquals(b, app.processingOrder.get(1)); - assertEquals(c, app.processingOrder.get(2)); - assertEquals(e, app.processingOrder.get(3)); - assertEquals(d, app.processingOrder.get(4)); - assertEquals(f, app.processingOrder.get(5)); - assertEquals(g, app.processingOrder.get(6)); - } - -} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java index e672763f13c..4683bccb0ad 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java @@ -75,6 +75,7 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { } private static void assertOrder(List<String> expectedSearchOrder, List<String> inputNames) { + inputNames.sort((a, b) -> a.compareTo(b)); Map<String, Schema> schemas = createSchemas(); List<Schema> inputSchemas = inputNames.stream() .map(schemas::get) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java index 209f339f51f..1498b9871d3 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Deployer.java @@ -54,7 +54,7 @@ public interface Deployer { */ Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout, boolean bootstrap); - /** Returns the time the current local active session was created, or empty if there is no local active session */ + /** Returns the time the current local active session was activated, or empty if there is no local active session */ Optional<Instant> lastDeployTime(ApplicationId application); /** Whether the deployer is bootstrapping, some users of the deployer will want to hold off with deployments in that case. */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index fd4f1824885..ca56a200c2c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -437,7 +437,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public Optional<Instant> lastDeployTime(ApplicationId application) { Tenant tenant = tenantRepository.getTenant(application.tenant()); if (tenant == null) return Optional.empty(); - return getActiveSession(tenant, application).map(Session::getCreateTime); + Optional<Instant> activatedTime = getActiveSession(tenant, application).map(Session::getActivatedTime); + log.log(Level.FINE, application + " last activated " + activatedTime.orElse(Instant.EPOCH)); + return activatedTime; } public ApplicationId activate(Tenant tenant, @@ -750,7 +752,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public HttpResponse getLogs(ApplicationId applicationId, Optional<DomainName> hostname, String apiParams) { String logServerURI = getLogServerURI(applicationId, hostname) + apiParams; - return logRetriever.getLogs(logServerURI); + return logRetriever.getLogs(logServerURI, lastDeployTime(applicationId)); } // ---------------- Methods to do call against tester containers in hosted ------------------------------ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java index 6d6b93dad46..0ec1382b496 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java @@ -24,7 +24,7 @@ public class ServerCache { private final ConfigDefinitionRepo builtinConfigDefinitions; private final ConfigDefinitionRepo userConfigDefinitions; - // NOTE: The reason we do a double mapping here is to de-dupe configs that have the same md5. + // NOTE: The reason we do a double mapping here is to de-dupe configs that have the same checksum. private final Map<ConfigCacheKey, PayloadChecksum> checksums = new ConcurrentHashMap<>(); private final Map<PayloadChecksum, ConfigResponse> checksumToConfig = new ConcurrentHashMap<>(); private final Object [] stripedLocks = new Object[113]; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 3cc3e749904..1aa70ff4b5b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -206,7 +206,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } else { deployLogger.log(Level.INFO, "Services did not converge on new config generation " + response.wantedGeneration + ", current generation: " + response.currentGeneration + ", will retry"); - try { Thread.sleep(10_000); } catch (InterruptedException e) { /* ignore */ } + try { Thread.sleep(5_000); } catch (InterruptedException e) { /* ignore */ } } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 30116b7ee69..d4be7bb7b31 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -380,7 +380,7 @@ public class ModelContextImpl implements ModelContext { private final Optional<CloudAccount> cloudAccount; public Properties(ApplicationId applicationId, - Version nodeVespaVersion, + Version modelVersion, ConfigserverConfig configserverConfig, Zone zone, Set<ContainerEndpoint> endpoints, @@ -394,7 +394,7 @@ public class ModelContextImpl implements ModelContext { SecretStore secretStore, List<X509Certificate> operatorCertificates, Optional<CloudAccount> cloudAccount) { - this.featureFlags = new FeatureFlags(flagSource, applicationId, nodeVespaVersion); + this.featureFlags = new FeatureFlags(flagSource, applicationId, modelVersion); this.applicationId = applicationId; this.multitenant = configserverConfig.multitenant() || configserverConfig.hostedVespa() || Boolean.getBoolean("multitenant"); this.configServerSpecs = fromConfig(configserverConfig); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index 14586e2289e..7d8aacba0d5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -6,8 +6,12 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.yolean.Exceptions; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; - import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; /** * @author olaaun @@ -16,11 +20,20 @@ public class LogRetriever { private final CloseableHttpClient httpClient = VespaHttpClientBuilder.create().build(); - public HttpResponse getLogs(String logServerUri) { + public HttpResponse getLogs(String logServerUri, Optional<Instant> deployTime) { HttpGet get = new HttpGet(logServerUri); try { return new ProxyResponse(httpClient.execute(get)); } catch (IOException e) { + // It takes some time before nodes are up after first-time deployment, return empty log for up to 1 minute + // if getting logs fail + if (deployTime.isPresent() && Instant.now().isBefore(deployTime.get().plus(Duration.ofMinutes(1)))) + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("".getBytes(StandardCharsets.UTF_8)); + } + }; return HttpErrorResponse.internalServerError("Failed to get logs: " + Exceptions.toMessageString(e)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java index 3f2cd1e41af..d0c4f43359a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java @@ -26,6 +26,15 @@ public class SessionHandler extends HttpHandler { } /** + * Set to make sure that timeout for the handler is higher than any timeouts used inside the handler (e.g. zookeeper barrier timeout) + * Setting this too low will lead to a response with status code 504 and empty response body. + */ + @Override + public Duration getTimeout() { + return Duration.ofSeconds(applicationRepository.configserverConfig().zookeeper().barrierTimeout()).plus(Duration.ofSeconds(30)); + } + + /** * Gets the raw session id from request (v2). Input request must have a valid path. * * @param request a request diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index b489eb70927..ad68073053d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -100,7 +100,7 @@ public class ApplicationApiHandler extends SessionHandler { @Override public Duration getTimeout() { - return zookeeperBarrierTimeout.plus(Duration.ofSeconds(10)); + return zookeeperBarrierTimeout.plus(Duration.ofSeconds(30)); } private TenantName validateTenant(HttpRequest request) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index 8f585dccf52..6244f806f47 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import java.time.Duration; - import com.yahoo.component.annotation.Inject; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.provision.ApplicationId; @@ -12,14 +10,15 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.ResponseHandler; -import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.http.v2.response.SessionActiveResponse; -import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.http.v2.response.SessionActiveResponse; +import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import java.time.Duration; +import java.util.logging.Level; /** * Handler that activates a session given by tenant and id (PUT). diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index 620f76961d4..7fd94667159 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -53,11 +53,6 @@ public class SessionPrepareHandler extends SessionHandler { return new SessionPrepareResponse(tenant.getName(), request, sessionId); } - @Override - public Duration getTimeout() { - return zookeeperBarrierTimeout.plus(Duration.ofSeconds(10)); - } - private Tenant getExistingTenant(HttpRequest request) { TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); Utils.checkThatTenantExists(tenantRepository, tenantName); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 468dda605c6..019b0386695 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -99,7 +99,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { Version wantedNodeVespaVersion) { log.log(Level.FINE, () -> String.format("Loading model version %s for session %s application %s", modelFactory.version(), applicationGeneration, applicationId)); - ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId, wantedNodeVespaVersion, applicationPackage); + ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId, modelFactory.version(), applicationPackage); Provisioned provisioned = new Provisioned(); ModelContext modelContext = new ModelContextImpl( applicationPackage, @@ -143,9 +143,11 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { return Optional.of(value); } - private ModelContext.Properties createModelContextProperties(ApplicationId applicationId, Version wantedNodeVespaVersion, ApplicationPackage applicationPackage) { + private ModelContext.Properties createModelContextProperties(ApplicationId applicationId, + Version modelVersion, + ApplicationPackage applicationPackage) { return new ModelContextImpl.Properties(applicationId, - wantedNodeVespaVersion, + modelVersion, configserverConfig, zone(), ImmutableSet.copyOf(new ContainerEndpointsCache(TenantRepository.getTenantPath(tenant), curator).read(applicationId)), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 51e61edfff3..e6ae5c491c0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -8,6 +8,8 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; @@ -22,6 +24,8 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -31,6 +35,7 @@ import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.FlagSource; import java.io.File; import java.io.IOException; @@ -38,6 +43,7 @@ import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.logging.Level; import java.util.logging.Logger; @@ -51,17 +57,24 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P private static final Logger log = Logger.getLogger(PreparedModelsBuilder.class.getName()); private final PermanentApplicationPackage permanentApplicationPackage; + private final FlagSource flagSource; + private final SecretStore secretStore; + private final List<ContainerEndpoint> containerEndpoints; + private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final ConfigDefinitionRepo configDefinitionRepo; private final HostValidator<ApplicationId> hostValidator; private final PrepareParams params; private final FileRegistry fileRegistry; private final Optional<ApplicationSet> currentActiveApplicationSet; - private final ModelContext.Properties properties; private final Curator curator; private final ExecutorService executor; public PreparedModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, PermanentApplicationPackage permanentApplicationPackage, + FlagSource flagSource, + SecretStore secretStore, + List<ContainerEndpoint> containerEndpoints, + Optional<EndpointCertificateSecrets> endpointCertificateSecrets, ConfigDefinitionRepo configDefinitionRepo, FileRegistry fileRegistry, ExecutorService executor, @@ -71,17 +84,20 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P DeployLogger deployLogger, PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, - ModelContext.Properties properties, - ConfigserverConfig configserverConfig) { - super(modelFactoryRegistry, configserverConfig, properties.zone(), hostProvisionerProvider, deployLogger); + ConfigserverConfig configserverConfig, + Zone zone) { + super(modelFactoryRegistry, configserverConfig, zone, hostProvisionerProvider, deployLogger); this.permanentApplicationPackage = permanentApplicationPackage; + this.flagSource = flagSource; + this.secretStore = secretStore; + this.containerEndpoints = containerEndpoints; + this.endpointCertificateSecrets = endpointCertificateSecrets; this.configDefinitionRepo = configDefinitionRepo; this.fileRegistry = fileRegistry; this.hostValidator = hostValidator; this.curator = curator; this.params = params; this.currentActiveApplicationSet = currentActiveApplicationSet; - this.properties = properties; this.executor = executor; } @@ -107,7 +123,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P new ApplicationCuratorDatabase(applicationId.tenant(), curator).readReindexingStatus(applicationId), createHostProvisioner(applicationPackage, provisioned), provisioned, - properties, + createModelContextProperties(modelFactory.version(), applicationPackage), getAppDir(applicationPackage), wantedDockerImageRepository, modelVersion, @@ -118,7 +134,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P } private ModelCreateResult createAndValidateModel(ModelFactory modelFactory, ApplicationId applicationId, Version modelVersion, ModelContext modelContext) { - log.log(properties.zone().system().isCd() ? Level.INFO : Level.FINE, + log.log(zone().system().isCd() ? Level.INFO : Level.FINE, () -> "Create and validate model " + modelVersion + " for " + applicationId + ", previous model is " + modelOf(modelVersion).map(Model::version).map(Version::toFullString).orElse("non-existing")); ValidationParameters validationParameters = @@ -139,7 +155,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P private HostProvisioner createHostProvisioner(ApplicationPackage applicationPackage, Provisioned provisioned) { HostProvisioner defaultHostProvisioner = DeployState.getDefaultModelHostProvisioner(applicationPackage); // Note: nodeRepositoryProvisioner will always be present when hosted is true - Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties.applicationId(), provisioned); + Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(params.getApplicationId(), provisioned); Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); if (allocatedHosts.isEmpty()) return nodeRepositoryProvisioner.orElse(defaultHostProvisioner); @@ -184,6 +200,25 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P throw exception; } + private ModelContext.Properties createModelContextProperties(Version modelVersion, + ApplicationPackage applicationPackage) { + return new ModelContextImpl.Properties(params.getApplicationId(), + modelVersion, + configserverConfig, + zone(), + Set.copyOf(containerEndpoints), + params.isBootstrap(), + currentActiveApplicationSet.isEmpty(), + LegacyFlags.from(applicationPackage, flagSource), + endpointCertificateSecrets, + params.athenzDomain(), + params.quota(), + params.tenantSecretStores(), + secretStore, + params.operatorCertificates(), + params.cloudAccount()); + } + /** The result of preparing a single model version */ public static class PreparedModelResult implements ModelResult { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 881d3de4eaa..edc166d0989 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -115,6 +115,10 @@ public abstract class Session implements Comparable<Session> { return sessionZooKeeperClient.readCreateTime(); } + public Instant getActivatedTime() { + return sessionZooKeeperClient.readActivatedTime(); + } + public void setApplicationId(ApplicationId applicationId) { sessionZooKeeperClient.writeApplicationId(applicationId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 4c1da9dc3af..1e073ac3458 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -26,6 +26,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.net.HostName; import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; @@ -61,6 +62,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig; + /** * A SessionPreparer is responsible for preparing a session given an application package. * @@ -157,7 +160,6 @@ public class SessionPreparer { final ContainerEndpointsCache containerEndpointsCache; final List<ContainerEndpoint> containerEndpoints; - final ModelContext.Properties properties; private final EndpointCertificateMetadataStore endpointCertificateMetadataStore; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<AthenzDomain> athenzDomain; @@ -191,24 +193,13 @@ public class SessionPreparer { .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); this.containerEndpoints = readEndpointsIfNull(params.containerEndpoints()); this.athenzDomain = params.athenzDomain(); - this.properties = new ModelContextImpl.Properties(params.getApplicationId(), - vespaVersion, - configserverConfig, - zone, - Set.copyOf(containerEndpoints), - params.isBootstrap(), - currentActiveApplicationSet.isEmpty(), - LegacyFlags.from(applicationPackage, flagSource), - endpointCertificateSecrets, - athenzDomain, - params.quota(), - params.tenantSecretStores(), - secretStore, - params.operatorCertificates(), - params.cloudAccount()); this.fileRegistry = fileDistributionFactory.createFileRegistry(serverDbSessionDir); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, + flagSource, + secretStore, + containerEndpoints, + endpointCertificateSecrets, configDefinitionRepo, fileRegistry, executor, @@ -218,8 +209,8 @@ public class SessionPreparer { logger, params, currentActiveApplicationSet, - properties, - configserverConfig); + configserverConfig, + zone); } void checkTimeout(String step) { @@ -236,7 +227,7 @@ public class SessionPreparer { FileDistribution fileDistribution = fileDistributionFactory.createFileDistribution(); log.log(Level.FINE, () -> "Ask other config servers to download application package for " + applicationId + " (" + fileReference + ")"); - properties.configServerSpecs() + ConfigServerSpec.fromConfig(configserverConfig) .stream() .filter(spec -> !spec.getHostName().equals(HostName.getLocalhost())) .forEach(spec -> fileDistribution.startDownload(spec.getHostName(), spec.getConfigServerPort(), Set.of(fileReference))); @@ -247,7 +238,7 @@ public class SessionPreparer { void preprocess() { try { - this.preprocessedApplicationPackage = applicationPackage.preprocess(properties.zone(), logger); + this.preprocessedApplicationPackage = applicationPackage.preprocess(zone, logger); } catch (IOException | RuntimeException e) { throw new IllegalArgumentException("Error preprocessing application package for " + applicationId + ", session " + sessionZooKeeperClient.sessionId(), e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 5603ef2df51..059d192e7d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -235,7 +235,7 @@ public class SessionRepository { public ConfigChangeActions prepareLocalSession(Session session, DeployLogger logger, PrepareParams params, Instant now) { params.vespaVersion().ifPresent(version -> { if ( ! params.isBootstrap() && ! modelFactoryRegistry.allVersions().contains(version)) - throw new UnknownVespaVersionException("Vespa version '" + version + "' not known by this configserver"); + throw new UnknownVespaVersionException("Vespa version '" + version + "' not known by this config server"); }); applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index de8cbcb4066..e146f0de187 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -33,7 +33,7 @@ import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; - +import org.apache.zookeeper.data.Stat; import java.security.cert.X509Certificate; import java.time.Instant; import java.util.List; @@ -239,6 +239,11 @@ public class SessionZooKeeperClient { return data.map(d -> Instant.ofEpochSecond(Long.parseLong(Utf8.toString(d)))).orElse(Instant.EPOCH); } + public Instant readActivatedTime() { + Optional<Stat> statData = curator.getStat(sessionStatusPath); + return statData.map(s -> Instant.ofEpochMilli(s.getMtime())).orElse(Instant.EPOCH); + } + private Path getCreateTimePath() { return sessionPath.append(CREATE_TIME_PATH); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index d9ca31d561d..ee2a979be7a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -7,7 +7,6 @@ import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.application.api.ApplicationMetaData; -import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; @@ -57,7 +56,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; - +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -75,6 +74,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.IntStream; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -245,10 +245,13 @@ public class ApplicationRepositoryTest { } @Test - public void getLogs() { + public void getLogs() throws IOException { deployApp(testAppLogServerWithContainer); HttpResponse response = applicationRepository.getLogs(applicationId(), Optional.empty(), ""); assertEquals(200, response.getStatus()); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + response.render(buffer); + assertEquals("log line", buffer.toString(UTF_8)); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java index 6a95c059fcc..4a521e3b8a2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java @@ -7,6 +7,8 @@ import com.yahoo.vespa.config.server.http.LogRetriever; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Optional; /** * @author olaa @@ -14,21 +16,14 @@ import java.nio.charset.StandardCharsets; public class MockLogRetriever extends LogRetriever { @Override - public HttpResponse getLogs(String logServerUri) { - return new MockHttpResponse(); - } - - private static class MockHttpResponse extends HttpResponse { - - private MockHttpResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("log line".getBytes(StandardCharsets.UTF_8)); - } - + public HttpResponse getLogs(String logServerUri, Optional<Instant>deployTime ) { + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("log line".getBytes(StandardCharsets.UTF_8)); + } + + }; } }
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index d4cf6324721..1f95e7fc75e 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -27,7 +27,7 @@ import java.util.logging.Logger; /** * A request handler which assigns a worker thread to handle each request. - * This is mean to be subclasses by handlers who does work by executing each + * This is meant to be subclassed by handlers who do work by executing each * request in a separate thread. * <p> * Note that this means that subclass handlers are synchronous - the request io can diff --git a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java index 569d1439bbe..4264492ca2b 100644 --- a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java +++ b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java @@ -47,8 +47,6 @@ public class GUIHandlerTest { public void testContentTypes() throws Exception{ assertResponse("/querybuilder/_includes/css/vespa.css", "/**","text/css; charset=UTF-8", 200); assertResponse("/querybuilder/js/agency.js", "/*!","application/javascript; charset=UTF-8", 200); - assertResponse("/querybuilder/img/reload.svg", "<?xml","image/svg+xml; charset=UTF-8", 200); - assertResponse("/querybuilder/img/Vespa-V2.png", null,"image/png; charset=UTF-8", 200); } @Test diff --git a/container-search-gui/src/main/resources/gui/_includes/css/vespa.css b/container-search-gui/src/main/resources/gui/_includes/css/vespa.css index a09364cefd4..19d5009e33f 100644 --- a/container-search-gui/src/main/resources/gui/_includes/css/vespa.css +++ b/container-search-gui/src/main/resources/gui/_includes/css/vespa.css @@ -8,9 +8,9 @@ --secondary-dark: #333; --muted: #777; - --fontprimary: HelveticaNeue,Helvetica,Arial,sans-serif; - --fontsecondary: HelveticaNeue-Thin,Helvetica,Arial,sans-serif; - }; + --fontprimary: "Hind Madurai",Helvetica,Arial,sans-serif; + --fontsecondary: "Hind Madurai",Helvetica,Arial,sans-serif; + } .bg-light-blue { background-image: linear-gradient(-1deg, rgba(63,157,216,0.08) 0%, rgba(163, 195, 215, 0.08) 97%); @@ -39,7 +39,7 @@ /*** Tooltips! ***/ .tip { visibility: visible; - border-bottom: 1px dotted [#FFFFFF]; + border-bottom: 1px dotted; position: relative; cursor: help; text-decoration: none; @@ -91,7 +91,7 @@ section h2.section-subheading, section h3.section-heading, section h3.section-subheading { text-transform: none; - font-family: HelveticaNeue-Thin,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; font-weight: normal; font-style: normal; } @@ -135,7 +135,7 @@ header .help-title { } .navbar-default { - background-color: #3F9DD8; + background-color: #005A8E; padding: 0; border-bottom: 2px solid rgba(255, 255, 255, 0.2); } @@ -146,14 +146,14 @@ header .help-title { } .navbar-default .navbar-brand { - background: transparent url("../../img/Vespa-V2.png") no-repeat; + background: transparent url("https://vespa.ai/assets/vespa-logo.png") no-repeat; background-size: contain; direction: ltr; text-indent: -9000px; height: 28px; width: 100px; display: inline-block; - font-family: HelveticaNeue,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; font-weight: bold; color: var(--primary); margin-top: 16px; @@ -166,7 +166,7 @@ header .help-title { letter-spacing: 1px; color: #FFFFFF; text-transform: none; - font-family: HelveticaNeue,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; } .navbar-default .navbar-nav > li > a:hover, @@ -189,7 +189,8 @@ section[id]:before { } header { - background-image: linear-gradient(0deg, #98C1DB 7%, #3F9DD8 100%); + /*background-image: linear-gradient(0deg, #98C1DB 7%, #3F9DD8 100%); */ + background-color: #005A8E; min-height: 1150px; } @@ -310,7 +311,7 @@ header .propvalue { margin-bottom: 3px; } -header .propvalue:-webkit-input-placeholder { /* Safari, Chrome(, Opera?) */ +header .propvalue::-webkit-input-placeholder { /* Safari, Chrome(, Opera?) */ color:gray; font-style:italic; } @@ -396,7 +397,6 @@ header .copyJSON{ height: 25px; border-width: 0px; border-radius: 5px; - padding: 0px padding-left: 1px; margin-top: 10px; margin-bottom: 20px; @@ -412,8 +412,7 @@ header .showJSON{ height: 25px; border-width: 0px; border-radius: 5px; - padding: 0px - padding-left: 1px; + padding: 0px; margin-top: 5px; margin-bottom: 10px; } @@ -426,13 +425,10 @@ header .pasteJSON{ height: 25px; border-width: 0px; border-radius: 5px; - padding: 0px; padding-left: 0px; padding-bottom: 2px; margin-top: 35px; margin-bottom: 10px; - - align-content: 30%; } @@ -444,7 +440,6 @@ header .copyURL{ height: 25px; border-width: 0px; border-radius: 5px; - padding: 0px padding-left: 1px; margin-top: 25px; margin-bottom: 10px; @@ -457,7 +452,7 @@ header .intro .intro-lead-in { font-size: 22px; line-height: 50px; margin-bottom: 25px; - font-family: HelveticaNeue-Thin,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; } header .intro .intro-heading { @@ -476,12 +471,12 @@ header .intro .btn-xl { color: #3F9DD8; font-weight: normal; text-transform: none; - font-family: HelveticaNeue,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; } header .intro-copy { - display:inline-block + display:inline-block; position: inherit; margin-top: 10px; padding-left: 50px; @@ -641,7 +636,7 @@ footer .credits { font-size: 12px; font-weight: normal; text-align: right; - font-family: HelveticaNeue-Thin,Helvetica,Arial,sans-serif; + font-family: "Hind Madurai",Helvetica,Arial,sans-serif; } footer .credits a { diff --git a/container-search-gui/src/main/resources/gui/_includes/index.html b/container-search-gui/src/main/resources/gui/_includes/index.html index bac486077a4..9a7c5e9ed33 100644 --- a/container-search-gui/src/main/resources/gui/_includes/index.html +++ b/container-search-gui/src/main/resources/gui/_includes/index.html @@ -1,15 +1,12 @@ <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <!DOCTYPE html> -<html> +<html lang="en"> <head> <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <title>Vespa. Big data. Real time.</title> + <title>Vespa Querybuilder</title> <meta name="viewport" content="width=device-width"> - <meta name="description" content=" Your big data processing and serving system - deep learning, - search applications, ad selection system, personalized - recommendation systems, and more..."> - + <meta name="description" content="Vespa Querybuilder - use to generate Vespa Query API requests"> <!-- Site icons - generated using http://realfavicongenerator.net/ --> <link rel="apple-touch-icon" sizes="180x180" href="/querybuilder/icons/apple-touch-icon.png"> @@ -27,161 +24,92 @@ <link href="https://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic" rel="stylesheet" type="text/css"> <link href='https://fonts.googleapis.com/css?family=Droid+Serif:400,700,400italic,700italic' rel='stylesheet' type='text/css'> <link href='https://fonts.googleapis.com/css?family=Roboto+Slab:400,100,300,700' rel='stylesheet' type='text/css'> + <link href='https://fonts.googleapis.com/css2?family=Hind+Madurai:wght@400;600;700' rel='stylesheet' type='text/css'> - <title>Vespa - Big Data. Real time.</title> - - <meta http-equiv="content-type" content="text/html; charset=utf-8"> - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - <meta name="description" content=""> + <meta http-equiv="content-type" content="text/html; charset=utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta name="description" content=""> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script> - - </head> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script> +</head> <body id="page-top" class="index"> - -<style type="text/css"></style> - - <link href="/querybuilder/_includes/css/agency.css" rel="stylesheet" type="text/css"> <link href="/querybuilder/_includes/css/bootstrap.min.css" rel="stylesheet" type="text/css"> <link href="/querybuilder/_includes/css/vespa.css" rel="stylesheet" type="text/css"> <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- Brand and toggle get grouped for better mobile display --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="https://vespa.ai">Vespa. Big data. Real time.</a> - </div> + <div class="container"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="https://vespa.ai">Vespa.ai</a> + </div> - <!-- Collect the nav links, forms, and other content for toggling --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav navbar-right"> - <li class="hidden"><a href="#page-top"></a> - <li><a href="https://blog.vespa.ai/">Blog</a> - <li><a href="https://twitter.com/vespaengine">Twitter</a> - <li><a href="https://docs.vespa.ai">Docs</a> - <li><a href="https://github.com/vespa-engine">GitHub</a> - <li><a href="https://docs.vespa.ai/en/getting-started.html">Get Started Now</a> - </ul> - </div> + <!-- Collect the nav links, forms, and other content for toggling --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right"> + <li><a href="https://docs.vespa.ai/en/query-api.html">Documentation</a></li> + <li><a href="https://github.com/vespa-engine/vespa/issues">Report issue</a></li> + </ul> </div> - </nav> + </div> +</nav> <!-- Header --> - <header> - <div class="layer"> - <div class="intro container"> - <div class="intro-lead-in">Vespa Search Engine</div> - <div class="intro-long">Select the method for sending requests and construct a query.</div> - <select class="methodselector" onchange="chooseMethod();" id="method"> - <option class="options" value="POST">POST</option> - <option class="options" value="GET">GET</option> - </select> - <input type="text" class="textbox" name="value" value="http://localhost:8080/search/" id="url" size="30"> - <button class="button" onclick="startSending();" id="send">Send</button> +<header> + <div class="layer"> + <div class="intro container"> + <div class="intro-lead-in">Vespa Querybuilder</div> + <div class="intro-long">Select the method for sending requests and construct a query:</div> + <select class="methodselector" onchange="chooseMethod();" id="method"> + <option class="options" value="POST">POST</option> + <option class="options" value="GET">GET</option> + </select> + <input type="text" class="textbox" name="value" value="http://localhost:8080/search/" id="url" size="30" /> + <button class="button" onclick="startSending();" id="send">Send</button> + + <br/><br/> - <br/> - <div id="request"> - </br> - <div class="intro-param">Construct a query by adding parameters or pasting a JSON.</div> - </div> - <br/> - <button class="addRow"onclick="addNewRow()" id="addRow">+</button> - <br/> - <button class="pasteJSON"onclick="pasteJSON();" id="pasteJSON"><img src="/querybuilder/img/paste.svg" height="16" width="16" style="margin-top:-2px; margin-right: 3px;"/> Paste JSON</button> - </br> - <button class="showJSON"onclick="showJSON();" id="showJSON">Show query JSON</button> - </br> - <textarea class="responsebox" id="jsonquery" cols=70 rows=15 style="display:none;"readonly></textarea> - </br> - <button class="copyJSON"onclick="copyToClipboard(jsonquery);" id="copyJSON"><img src="/querybuilder/img/copy.svg" height="17" width="17" /> Copy as JSON</button> - <button class="copyURL"onclick="copyURL();" id="copyURL"><img src="/querybuilder/img/copy.svg" height="17" width="17" /> Copy as URL</button> - <br/> + <div id="request"> + <div class="intro-param">Construct a query by adding parameters or pasting a JSON:</div> + </div> + <br/> + <button class="addRow" onclick="addNewRow()" id="addRow">+</button> + <br/> + <button class="pasteJSON" onclick="pasteJSON();" id="pasteJSON"> + <img src="/querybuilder/img/paste.svg" + height="16" width="16" style="margin-top:-2px; margin-right: 3px;" alt="paste"/> + Paste JSON + </button> + <button class="showJSON" onclick="showJSON();" id="showJSON">Show query JSON</button> + <textarea class="responsebox" id="jsonquery" cols=70 rows=15 style="display:none;"readonly></textarea> + <button class="copyJSON" onclick="copyToClipboard(jsonquery);" id="copyJSON"> + <img src="/querybuilder/img/copy.svg" height="17" width="17" alt="Copy as JSON"/> Copy as JSON + </button> + <button class="copyURL" onclick="copyURL();" id="copyURL"> + <img src="/querybuilder/img/copy.svg" height="17" width="17" alt="Copy as URL"/> Copy as URL + </button> + <br/> - <div class="response">Response</div> - <input class="responseinfo" id="reponsestatus" value="" readonly></input></br> - <textarea class="responsebox" id="response" cols=70 rows=25 readonly></textarea> - <div id="div" > - <div > + <div class="response">Response:</div> + <input class="responseinfo" id="reponsestatus" value="" readonly /> + <textarea class="responsebox" id="response" cols=70 rows=25 readonly></textarea> + <div id="div" > + <div> <div class="intro-copy" onclick="copyToClipboard(response);" style="cursor:pointer;"> - <img src="/querybuilder/img/copy.svg" height="30" width="30" /> - </div> - <div class="intro-refresh" onclick="refresh();" style="cursor:pointer;"> - <img src="/querybuilder/img/reload.svg" height="30" width="30" /> - </div> - </br> - </br> - <div id="helpbutton" class="intro-help" onclick="toggleHelp();" style="cursor:pointer;"> - <img src="/querybuilder/img/features-help.png" height="47" width="138" style="margin-left: -1px;"/> + <img src="/querybuilder/img/copy.svg" height="30" width="30" alt="Copy to clipboard"/> </div> - <div id="help" style="display: none;"> - <div class="intro-param" id="help"> - </br> - <div class="help-title">Features</div> - <span> ○ Autocompletion of YQL-syntax</span> </br> - <span> ○ Drop-down lists of all valid parameters</span> </br> - <span> ○ Sending both POST and GET-requests to <i>Vespa</i></span> </br> - <span> ○ Easy access to the <a href="https://docs.vespa.ai/en/reference/query-api-reference.html">documentation</a> of each parameter</span> </br> - <span> ○ Conversion of POST- to GET-query</span> </br> - <span> ○ Pasting already built JSON-query</span> </br> - <span> ○ View and copy the response of queries</span> </br> - <span> ○ View the finished JSON-query as you build </span> </br> - </div> - </br> - <div class="intro-param" id="help"> - <div class="help-title">Help</div> - <span>Please submit a <a href="https://github.com/vespa-engine/vespa/blob/master/container-search-gui/src/main/resources/gui/_includes/index.html">pull request</a> or create an <a href="https://github.com/vespa-engine/vespa/issues">issue</a> for fixes to the querybuilder.</span> </br> - </div> - </div> - </div> - </div> + </div> </div> </div> - </header> - <footer> - <div class="container"> - <div class="row"> - <div class="col-xs-2 quicklink-section"> - <div class="footer-title">Resources</div> - <ul class="quicklinks"> - <li><a href="https://docs.vespa.ai/en/vespa-quick-start.html">Getting Started</a> - <li><a href="https://docs.vespa.ai">Documentation</a> - <li><a href="https://github.com/vespa-engine/vespa">Open source</a> - </ul> - </div> - <div class="col-xs-2 quicklink-section"> - <div class="footer-title">Contact</div> - <ul class="quicklinks"> - <li><a href="https://twitter.com/vespaengine">Twitter</a> - <li><a href="mailto:info@vespa.ai">info@vespa.ai</a> - <li><a href="https://github.com/vespa-engine/vespa/issues">Issues</a> - </ul> - </div> - <div class="col-xs-2 quicklink-section"> - <div class="footer-title">Community</div> - <ul class="quicklinks"> - <li><a href="https://github.com/vespa-engine/vespa/blob/master/CONTRIBUTING.md">Contributing</a> - <li><a href="https://stackoverflow.com/questions/tagged/vespa">Stack Overflow</a> - <li><a href="https://gitter.im/vespa-engine/Lobby">Gitter</a> - </ul> - </div> - <div class="col-xs-6 quicklink-section"> - <div class="credits"> - <span>Copyright Yahoo</span> - Licensed under <a href="https://github.com/vespa-engine/vespa/blob/master/LICENSE">Apache License 2.0</a> - , <a href="https://github.com/y7kim/agency-jekyll-theme">Theme</a> by Rick K. - </div> - </div> - </div> - </div> - </footer> + </div> +</header> <!-- jQuery Version 1.11.0 --> <script src="/querybuilder/js/jquery-1.11.0.js"></script> @@ -229,7 +157,6 @@ childrenProps["ranking.properties"] = window.CONFIG.ranking_properties; } addNewRow(); - getSearchApiReference(); }, 250); }; var stringType = ["yql", "queryProfile", "searchChain", "model.defaultIndex", "model.encoding", "model.language", @@ -247,19 +174,6 @@ var yqlID = "v1"; - - function toggleHelp(){ - var div = document.getElementById("help"); - var buttonDiv = document.getElementById("helpbutton"); - if (div.style.display === "none") { - div.style.display = "block"; - buttonDiv.style.backgroundColor = '#4EA2D6'; - } else { - div.style.display = "none"; - buttonDiv.style.backgroundColor = 'transparent'; - } - } - function updateFields(){ var temp = number; while (temp > 0){ @@ -449,6 +363,7 @@ newButton.classList.add("removeRow"); var br = document.createElement("br"); br.id = "br"+temp + /* var img = document.createElement("img"); img.src = "/querybuilder/img/information.svg"; img.height = "15"; @@ -463,11 +378,12 @@ a.id = "inf"+temp a.appendChild(img); a.appendChild(span); + */ var div = document.createElement("div"); div.id = number; div.appendChild(newInput); div.appendChild(newDatalist); - div.appendChild(a); + //div.appendChild(a); div.appendChild(newInputVal); div.appendChild(newDatalist2); div.appendChild(newButton); @@ -490,6 +406,7 @@ generateJSON(); return temp; } + /* ToDo: remove completely function showInformation(no, key){ var a = document.getElementById("inf"+no); if(validKey(no, key)){ @@ -504,6 +421,7 @@ a.style = "visibility: hidden;" } } + */ function validKey(no, possibleKey){ if (contains(possible, possibleKey)){return true;} for (var key in childrenProps){ @@ -528,7 +446,7 @@ } function keySelected(no, value){ var key = document.getElementById("i"+no).value; - showInformation(no, key); + //showInformation(no, key); ToDo: remove completely findUsedProps(); var fullKey = getFullName(no, key); if(fullKey in childrenProps){ @@ -728,6 +646,7 @@ b.innerHTML = ' ↳ '; var margin = 20*(temp).length; b.style = "padding-left:"+margin+"px;"; + /* var img = document.createElement("img"); img.src = "/querybuilder/img/information.svg"; img.height = "15"; @@ -742,10 +661,11 @@ a.id = "inf"+temp a.appendChild(img); a.appendChild(span); + */ div.appendChild(b); div.appendChild(newInput); div.appendChild(newDatalist); - div.append(a); + //div.append(a); div.appendChild(newInputVal); div.appendChild(newDatalist2); div.appendChild(newButton); @@ -789,9 +709,7 @@ } return false; } - function refresh(){ - document.location.reload(true); - } + function generateJSON(){ json = JSON.parse("{}"); buildJSON(json, number, 0); @@ -940,20 +858,6 @@ copyURL.style.display = "none"; } } - function getSearchApiReference(){ - var div = document.getElementById("div"); - var object = document.createElement("object"); - object.data = "/querybuilder/_includes/search-api-reference.html"; - object.type ="text/html"; - object.style = "visibility: hidden; height: 1px; width: 200px;" - div.appendChild(object); - setTimeout(function(){ - searchApiReference = object.contentDocument; - setTimeout(function(){ - div.removeChild(object); - }, 20); - }, 150); - } function changeInformation(no,key){ if (key===""){return "Choose a parameter for information"} if (getFullName(no,key) in childrenProps){return "Add parameters under the parent ''" + getFullName(no,key) + "'"} diff --git a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html deleted file mode 100644 index 086ff293b14..00000000000 --- a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html +++ /dev/null @@ -1,1914 +0,0 @@ - -<!DOCTYPE html> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<html lang="en"> - -<head> - <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <!-- <meta name="viewport" content="width=device-width, initial-scale=1"> --> - <meta name="description" content="Serve big data with ease - dynamic responses at any scale in milliseconds with any traffic volume."> - <meta name="author" content="Vespa team"> - - <title>Search API Reference</title> - - <!-- Bootstrap --> - <link href="/css/bootstrap.min.css" rel="stylesheet"> - - <!-- Font Awesome --> - <link rel="stylesheet" href="/css/font-awesome.min.css"> - - <!-- Docs layout --> - <link rel="stylesheet" href="/css/docs.css"> - - <!-- Favicons --> - <link rel="icon" href="/icons/favicon.ico" type="image/x-icon" /> - <link rel="shortcut icon" href="/icons/favicon.ico" type="image/x-icon" /> - - <!-- Bootstrap and other JavaScript --> - <script type="text/javascript" async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-MML-AM_CHTML"></script> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script> - <script src="/js/bootstrap.min.js"></script> - - <!-- Global Site Tag (gtag.js) - Google Analytics --> - <script async src="https://www.googletagmanager.com/gtag/js?id=UA-107187180-3"></script> - <script> - window.dataLayer = window.dataLayer || []; - function gtag(){dataLayer.push(arguments)}; - gtag('js', new Date()); - gtag('config', 'UA-107187180-3'); -</script> - -</head> - -<body> - -<!-- Fixed navbar --> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<nav class="navbar navbar-default navbar-fixed-top"> - <div class="container-fluid"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="http://vespa.ai"> - <img src="/img/vespa-logo.png" width="100" height="28"> - </a> - </div> - <div id="navbar" class="navbar-collapse collapse"> - <ul class="nav navbar-nav navbar-right"> - <li><a href="http://blog.vespa.ai/">Blog</a> - <li><a href="https://twitter.com/vespaengine">Twitter</a> - <li><a href="/documentation/">Docs</a> - <li><a href="https://github.com/vespa-engine" target="_blank">GitHub</a> - <li><a href="https://github.com/vespa-engine/vespa/issues" target="_blank">Issues</a> - </ul> - <div class="col-sm-offset-2 col-md-offset-2"> - <div class="row"> - <form class="search" action="/search.html" method="get" id="search-form"> - <div class="col-xs-4"> - <input type="text" class="form-control" placeholder="Search Documentation" id="search" name="q"> - </div> - <button type="submit" id="submit" class="btn btn-search"><i class="fa fa-search" aria-hidden="true"></i></button> - </form> - </div> - </div> - </div> - </div> -</nav> - - -<div id="main-content" class="container-fluid"> - <div class="row"> - - <!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> - - <div class="col-sm-3 col-md-2 sidebar"> - <ul class="nav nav-sidebar"> - <li class="collapse-all" onclick="expand_all();">[+] expand all</li> - </ul> - - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">WELCOME</li> - - - - <li class="collapseable"><a href="/">Welcome to Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/overview.html">Vespa Overview</a></li> - - - - - <li class="collapseable"><a href="/documentation/features.html">Features</a></li> - - - - - <li class="collapseable"><a href="/documentation/elastic-search-comparison.html">Comparison to Elasticsearch</a></li> - - - - - <li class="collapseable"><a href="/documentation/contributing.html">Contributing to Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/introduction-to-documentation.html">Documentation Conventions</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">GETTING STARTED</li> - - - - <li class="collapseable"><a href="/documentation/vespa-quick-start.html">Quick Start</a></li> - - - - - <li class="collapseable"><a href="/documentation/tutorials/blog-search.html">Blog Search Tutorial</a></li> - - - - - <li class="collapseable"><a href="/documentation/build-install-vespa.html">Build Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/api.html">Vespa API and Interfaces</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">DOCUMENTS AND SEARCH DEFINITIONS</li> - - - - <li class="collapseable"><a href="/documentation/documents.html">Documents</a></li> - - - - - <li class="collapseable"><a href="/documentation/search-definitions.html">Search Definitions</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">WRITING TO VESPA</li> - - - - <li class="collapseable"><a href="/documentation/writing-to-vespa.html">Writing to Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/vespa-feed-client.html">Vespa Feeding Client API</a></li> - - - - - <li class="collapseable"><a href="/documentation/feed-using-hadoop-pig-oozie.html">Feed using Hadoop, Pig, Oozie</a></li> - - - - - <li class="collapseable"><a href="/documentation/document-api-guide.html">Document API</a></li> - - - - - <li class="collapseable"><a href="/documentation/document-processing-overview.html">Document Processing</a></li> - - - - - <li class="collapseable"><a href="/documentation/annotations.html">Annotations API</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">QUERYING VESPA</li> - - - - <li class="collapseable"><a href="/documentation/querying-vespa.html">Querying Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/search-api.html">Search API</a></li> - - - - - <li class="collapseable"><a href="/documentation/query-language.html">Vespa Query Language</a></li> - - - - - <li class="collapseable"><a href="/documentation/grouping.html">Grouping Information in Results</a></li> - - - - - <li class="collapseable"><a href="/documentation/federation.html">Federation</a></li> - - - - - <li class="collapseable"><a href="/documentation/query-profiles.html">Query Profiles</a></li> - - - - - <li class="collapseable"><a href="/documentation/geo-search.html">Geo Search</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">RANKING AND ML MODELS</li> - - - - <li class="collapseable"><a href="/documentation/ranking.html">Ranking Introduction</a></li> - - - - - <li class="collapseable"><a href="/documentation/nativerank.html">Ranking With nativeRank</a></li> - - - - - <li class="collapseable"><a href="/documentation/tensor-intro.html">Tensor Introduction</a></li> - - - - - <li class="collapseable"><a href="/documentation/tensor-user-guide.html">Tensor User Guide</a></li> - - - - - <li class="collapseable"><a href="/documentation/tensorflow.html">Ranking With TensorFlow Models</a></li> - - - - - <li class="collapseable"><a href="/documentation/onnx.html">Ranking With ONNX Models</a></li> - - - - - <li class="collapseable"><a href="/documentation/xgboost.html">Ranking With XGBoost Models</a></li> - - - - - <li class="collapseable"><a href="/documentation/stateless-model-evaluation.html">Stateless model evaluation</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">LINGUISTICS AND TEXT PROCESSING</li> - - - - <li class="collapseable"><a href="/documentation/linguistics.html">Linguistics in Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/stemming.html">Stemming</a></li> - - - - - <li class="collapseable"><a href="/documentation/text-processing.html">Special Tokens</a></li> - - - - - <li class="collapseable"><a href="/documentation/query-rewriting.html">Query Rewriting</a></li> - - - - - <li class="collapseable"><a href="/documentation/query-phrasing.html">Query Phrasing With PhrasingSearcher</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">DEVELOPING APPLICATIONS AND PLUGINS</li> - - - - <li class="collapseable"><a href="/documentation/cloudconfig/application-packages.html">Application Packages</a></li> - - - - - <li class="collapseable"><a href="/documentation/container-intro.html">The Vespa Container</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/">Java Data Intensive Serving Container - JDisc</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/container-components.html">JDisc Container Components</a></li> - - - - - <li class="collapseable"><a href="/documentation/component-types.html">Container Component Types</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/developing-applications.html">Application Development Basics</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/processing.html">Request-Response Processing</a></li> - - - - - <li class="collapseable"><a href="/documentation/searcher-development.html">Searcher Development</a></li> - - - - - <li class="collapseable"><a href="/documentation/docproc-development.html">Document Processor Development</a></li> - - - - - <li class="collapseable"><a href="/documentation/developing-web-services.html">Developing Web Service Applications</a></li> - - - - - <li class="collapseable"><a href="/documentation/handler-tutorial.html">HTTP API Use Case Tutorial</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/injecting-components.html">Component Injection</a></li> - - - - - <li class="collapseable"><a href="/documentation/chained-components.html">Chained Components</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/developing-osgi-bundles.html">Building OSGi Bundles</a></li> - - - - - <li class="collapseable"><a href="/documentation/bundle-plugin.html">OSGi Bundle Maven Plugin</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/component-versioning.html">Versioning in the Container</a></li> - - - - - <li class="collapseable"><a href="/documentation/jdisc/metrics.html">Container Metrics</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">CONFIGURATION</li> - - - - <li class="collapseable"><a href="/documentation/cloudconfig/config-introduction.html">The Cloud Configuration System</a></li> - - - - - <li class="collapseable"><a href="/documentation/cloudconfig/configapi-dev.html">Cloud Config API</a></li> - - - - - <li class="collapseable"><a href="/documentation/cloudconfig/cloudconfig-model-plugins.html">Developing Cloud Config Model Plugins</a></li> - - - - - <li class="collapseable"><a href="/documentation/cloudconfig/deploy-rest-api-v2.html">Deploy API</a></li> - - - - - <li class="collapseable"><a href="/documentation/cloudconfig/config-rest-api-v2.html">Config API</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">RESULT FORMATS</li> - - - - <li class="collapseable"><a href="/documentation/document-summaries.html">Document Summaries</a></li> - - - - - <li class="collapseable"><a href="/documentation/result-rendering.html">Search Result Renderers</a></li> - - - - - <li class="collapseable"><a href="/documentation/page-templates.html">Page Templates</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">PREDICATE SEARCH</li> - - - - <li class="collapseable"><a href="/documentation/predicate-fields.html">Predicate Fields</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">USER AND GROUP ORIENTED SEARCH</li> - - - - <li class="collapseable"><a href="/documentation/streaming-search.html">Streaming Search</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">THE CONTENT LAYER</li> - - - - <li class="collapseable"><a href="/documentation/elastic-vespa.html">Elastic Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/proton.html">Proton</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/consistency.html">Vespa Consistency Model</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/idealstate.html">Distribution Algorithm</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/data-placement.html">Document Distribution</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/buckets.html">Buckets</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/admin-states.html">Cluster and Node States</a></li> - - - - - <li class="collapseable"><a href="/documentation/content/api-state-rest-api.html">State API</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">PERFORMANCE AND TUNING</li> - - - - <li class="collapseable"><a href="/documentation/performance/sizing-search.html">Vespa Search Sizing Guide</a></li> - - - - - <li class="collapseable"><a href="/documentation/attributes.html">Document Attributes</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/attribute-memory-usage.html">Attribute Memory Usage</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/vespa-benchmarking.html">Vespa Benchmarking</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/profiling-search-container.html">Profiling the Search Container</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/container-tuning.html">Container Tuning</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/rate-limiting-searcher.html">Rate-Limiting Search Requests</a></li> - - - - - <li class="collapseable"><a href="/documentation/performance/fbench.html">Vespa HTTP Benchmark Tool - vespa-fbench</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">OPERATIONS AND PROCEDURES</li> - - - - <li class="collapseable"><a href="/documentation/operations/admin-procedures.html">Administrative Procedures</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/vespa-cmdline-tools.html">Vespa Command-line Tools</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/metrics-health-format.html">Gathering Metrics from Vespa</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/logs.html">Log Files</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/files-processes-and-ports.html">Files, Processes and Ports</a></li> - - - - - <li class="collapseable"><a href="/documentation/inspecting-java-services.html">Inspecting Vespa Java Services</a></li> - - - - - <li class="collapseable"><a href="/documentation/docker-containers-in-production.html">Docker Containers in Production</a></li> - - - - - <li class="collapseable"><a href="/documentation/securing-your-vespa-installation.html">Securing a Vespa installation</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">MIGRATING FROM ELASTICSEARCH TO VESPA</li> - - - - <li class="collapseable"><a href="/documentation/migrating-from-elastic-search-to-vespa.html">Migrating from Elasticsearch to Vespa</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">CONFIGURATION REFERENCE</li> - - - - <li class="collapseable"><a href="/documentation/reference/application-packages-reference.html">Application Package Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/search-definitions-reference.html">Search Definition Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/services.html">Services Configuration (services.xml)</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/advanced-indexing-language.html">Indexing Language</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/config-files.html">Custom Configuration File Reference</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">RANKING REFERENCE</li> - - - - <li class="collapseable"><a href="/documentation/reference/ranking-expressions.html">Ranking Expressions</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/tensor.html">Tensor Evaluation Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/nativerank.html">nativeRank Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/rank-features.html">Rank Feature Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/string-segment-match.html">String Segment Match</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/rank-feature-configuration.html">Rank Feature Configuration</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/rank-types.html">Rank Types</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">QUERIES AND RESULTS REFERENCE</li> - - - - <li class="active collapseable"><a href="/documentation/reference/search-api-reference.html">Search API Reference <span class="sr-only">(current)</span></a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/simple-query-language-reference.html">Simple Query Language Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/select-reference.html">Select Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/grouping-syntax.html">Grouping Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/sorting.html">Sorting Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/query-profile-reference.html">Query Profile Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/semantic-rules.html">Semantic Rule Language Reference</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/advanced-search-operators.html">Advanced Search Operators</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/equiv.html">The EQUIV Operator</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/wand-operator.html">The WAND Operator</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/weighted-set-term.html">The WeightedSetItem Operator</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/dot-product-search-operator.html">The Dot Product Search Operator</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/default-result-format.html">Default JSON Result Format</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/page-templates-syntax.html">Page Templates Syntax</a></li> - - - - - </ul> - - <ul class="nav nav-sidebar"> - <li class="collapse-parent" onclick="on_collapse(this);">DOCUMENT FORMAT AND LANGUAGES REFERENCE</li> - - - - <li class="collapseable"><a href="/documentation/reference/document-json-format.html">Document JSON Format</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/document-field-path.html">Document Field Path Syntax</a></li> - - - - - <li class="collapseable"><a href="/documentation/reference/document-select-language.html">Document Selector Language</a></li> - - - - - </ul> - - - </div> - - <script type="application/javascript"> - - $(document).ready (function() { - $(".collapseable").each(function() { - $(this).addClass("collapsed"); - }); - $(".active").each(function() { - $(this).removeClass("collapsed"); - $(this).siblings().each(function() { - if (!$(this).hasClass("collapse-parent")) { - $(this).removeClass("collapsed"); - } - }); - }); - }); - - function on_collapse(e) { - $(e).siblings().each(function() { - $(this).toggleClass("collapsed"); - }); - } - - function expand_all() { - $(".collapseable").each(function() { - $(this).removeClass("collapsed"); - }); - } - -</script> - - - <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main"> - - <h1> - Search API Reference - <div class="pull-right"><a href="https://github.com/vespa-engine/documentation/blob/master/documentation/reference/search-api-reference.html" class="btn btn-link"><i class="fa fa-github"></i> edit page</a></div> - </h1> - - <p> - All the search request parameters listed below can be set in query - profiles. The first four blocks of properties are also modeled as - query profile types. These types can be referred from query profiles - (and inheriting types) to provide type checking on the parameters. - </p><p> - These parameters often have both a full name - which includes the - path from the root query profile - and one or more abbreviated - names. Both names can be used in search requests, while only full - names can be used in query profiles. The full names are case - sensitive, while the abbreviated names are case insensitive. - </p><p> - The parameters modeled as query profiles are also available through - get methods as Java objects from the Query to Searcher components. - </p> - - - - <h2>Index</h2> - - <dt>Query</dt> - <dd> - <ul> - <li><a href="#yql">yql</a></li> - <li><a href="#Select">select</a></li> - <ul> - <li><a href="#where">where</a></li> - <li><a href="#grouping">grouping</a></li> - </ul> - </ul> - </dd> - </dl> - - <dl> - <dt>Native Execution Parameters</dt> - <dd> - <ul> - <li><a href="#hits">hits</a> [<em>count</em>]</li> - <li><a href="#offset">offset </a>[<em>start</em>]</li> - <li><a href="#queryProfile">queryProfile</a></li> - <li><a href="#nocache">nocache</a></li> - <li><a href="#groupingSessionCache">groupingSessionCache</a></li> - <li><a href="#searchChain">searchChain</a></li> - <li><a href="#timeout">timeout</a></li> - <li><a href="#tracelevel">tracelevel</a></li> - <li><a href="#trace.timestamps">trace.timestamps</a></li> - </ul> - </dd> - - <dt>Query Model Parameters</dt> - <dd> - <ul> - <li><a href="#model.defaultIndex">model.defaultIndex</a> [<em>default-index</em>]</li> - <li><a href="#model.encoding">model.encoding</a> [<em>encoding</em>]</li> - <li><a href="#model.filter">model.filter</a> [<em>filter</em>]</li> - <li><a href="#model.language">model.language</a> [<em>lang, language</em>]</li> - <li><a href="#model.queryString">model.queryString</a> [<em>query</em>]</li> - <li><a href="#model.restrict">model.restrict</a> [<em>restrict</em>]</li> - <li><a href="#model.searchPath">model.searchPath</a> [<em>path</em>]</li> - <li><a href="#model.sources">model.sources</a> [<em>search, sources</em>]</li> - <li><a href="#model.type">model.type</a> [<em>type</em>]</li> - </ul> - </dd> - - <dt>Ranking</dt> - <dd> - <ul> - <li><a href="#ranking.location">ranking.location</a> [<em>location</em>]</li> - <li><a href="#ranking.features">ranking.features</a> [<em>rankfeature</em>]</li> - <li><a href="#ranking.listFeatures">ranking.listFeatures</a> [<em>rankfeatures</em>]</li> - <li><a href="#ranking.profile">ranking.profile</a> [<em>ranking</em>]</li> - <li><a href="#ranking.properties">ranking.properties</a> [<em>rankproperty</em>]</li> - <li><a href="#ranking.sorting">ranking.sorting</a> [<em>sorting</em>]</li> - <li><a href="#ranking.freshness">ranking.freshness</a></li> - <li><a href="#ranking.queryCache">ranking.queryCache</a></li> - <li><a href="#ranking.matchPhase">ranking.matchPhase</a></li> - </ul> - </dd> - - <dt>Presentation</dt> - <dd> - <ul> - <li><a href="#presentation.bolding">presentation.bolding</a> [<em>bolding</em>]</li> - <li><a href="#presentation.format">presentation.format</a> [<em>format</em>]</li> - <li><a href="#presentation.template">presentation.template</a></li> - <li><a href="#presentation.summary">presentation.summary</a> [<em>summary</em>]</li> - <li><a href="#presentation.timing">presentation.timing</a></li> - </ul> - </dd> - - <dt>Grouping</dt> - <dd> - <ul> - <li><a href="#select">select</a></li> - <li><a href="#collapse.summary">collapse.summary</a></li> - <li><a href="#collapsefield">collapsefield</a></li> - <li><a href="#collapsesize">collapsesize</a></li> - </ul> - </dd> - - <dt>Geographical Searches</dt> - <dd> - <ul> - <li><a href="#pos.ll">pos.ll</a></li> - <li><a href="#pos.radius">pos.radius</a>,</li> - <li><a href="#pos.attribute">pos.attribute</a></li> - <li><a href="#pos.bb">pos.bb</a></li> - </ul> - </dd> - - <dt>Streaming Search</dt> - <dd> - <ul> - <li><a href="#streaming.userid">streaming.userid</a></li> - <li><a href="#streaming.groupname">streaming.groupname</a></li> - <li><a href="#streaming.selection">streaming.selection</a></li> - <li><a href="#streaming.priority">streaming.priority</a></li> - <li><a href="#streaming.maxbucketspervisitor">streaming.maxbucketspervisitor</a></li> - </ul> - </dd> - - <dt>Semantic Rules</dt> - <dd> - <ul> - <li><a href="#rules.off">rules.off</a></li> - <li><a href="#rules.rulebase">rules.rulebase</a></li> - <li><a href="#tracelevel.rules">tracelevel.rules</a></li> - </ul> - </dd> - - <dt>Other</dt> - <dd> - <ul> - <li><a href="#recall">recall</a></li> - <li><a href="#user">user</a></li> - <li><a href="#nocachewrite">nocachewrite</a></li> - <li><a href="#hitcountestimate">hitcountestimate</a></li> - <li><a href="#metrics.ignore">metrics.ignore</a></li> - </ul> - </dd> - </dl> - - - <h2 id="query">Query</h2> - <h3 id="yql">yql</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>String</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - The YQL query will be parsed and executed in the backend. - Only simple YQL programs are supported, refer to - <a href="../query-language.html">YQL</a> for details. - </p> - - <h3 id="Select">select</h3> - <p>Select query is equivalent with YQL, written in JSON. Contains subparameters <code>where</code> and <code>grouping</code>.</p> - - <h4 id="where"> where</h4> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>JSON</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - - <h4 id="grouping"> grouping</h4> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>JSON</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - The where and grouping query will be parsed and executed in the backend. - Refer to - <a href="../reference/select-reference.html">Select Reference</a> for details. - </p> - - - - - <h2 id="native-execution-parameters">Native Execution Parameters</h2> - <p> - These parameters are defined in the <code>native</code> query profile type. - </p> - - - <h3 id="hits">hits</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>count</td></tr> - <tr><td>Values</td> - <td> - A positive integer, or 0. The sum of <a href="#offset">offset</a> and - <a href="#hits">hits</a> should be lower than the configured maxoffset - value, and will be adjusted to fit. See also comment - at <code>offset</code>. - </td> - </tr> - <tr><td>Default</td><td>10</td></tr> - </table> - <p> - The maximum number of hits to return from the result set. - Must be lower than <code>maxHits</code>, which is either set in a - <a href="#queryProfile">query profile</a>, or default 400. - <!-- ToDo: link to def file or code where this is definied --> - </p> - - - <h3 id="offset">offset</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>start</td></tr> - <tr><td>Values</td> - <td> - A positive integer, including 0. - </td> - </tr> - <tr><td>Default</td><td>0</td></tr> - </table> - <p> - The index of the first hit to return from the result set. - Must be lower than <code>maxOffset</code>, which is either set in a - <a href="#queryProfile">query profile</a>, or default 1000. - <!-- ToDo: link to def file or code where this is definied --> - </p> - - - <h3 id="queryProfile">queryProfile</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td><em>None</em></td></tr> - <tr><td>Values</td> - <td> - A query profile id - name:version, where version can be omitted - or partially specified, e.g "myprofile:2.1" - </td> - </tr> - <tr><td>Default</td><td><code>default</code></td></tr> - </table> - <p> - A <a href="../query-profiles.html">query profile</a> has default properties for a query. - The default query profile is named <em>default</em> - example: - <pre> -<query-profile id="default"> - <field name="maxHits">10</field> - <field name="maxOffset">1000</field> -</query-profile> -</pre> - </p> - - - <h3 id="nocache">nocache</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - True or false - </td> - </tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Set to true to avoid the result being fetched from cache, and avoid - writing the result to cache after fetching it. - </p> - - - <h3 id="groupingSessionCache">groupingSessionCache</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - True or false - </td> - </tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Set to true to store intermediate grouping results in the search back ends when - using multi level grouping expressions in order to speed up grouping at a - potential loss of accuracy. See the <a - href="grouping-syntax.html#sessionCache">grouping reference</a> for more - details. - </p> - - - <h3 id="searchChain">searchChain</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - A search chain id - name:version, where version can be - omitted or partially specified, e.g "mychain:2.1.3". - </td> - </tr> - <tr><td>Default</td><td><code>default</code></td></tr> - </table> - <p> - The search chain initially invoked when processing this query. This - search chain may invoke other chains. - </p> - - - <h3 id="timeout">timeout</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - Positive floating point number with an optional unit. Default unit - is seconds (s), valid unit strings are e.g. <em>ms</em> and <em>s</em>. To set - a timeout of one minute, the argument could be set to <em>60 s</em>. - Space between the number and the unit is optional. - </td> - </tr> - <tr><td>Default</td><td>Undefined, but guaranteed to be at least 5000 milliseconds. This default can be overridden by configuring timeout in a <a href="../query-profiles.html">query profile.</a></td></tr> - </table> - <p>The query timeout.</p> - - - <h3 id="tracelevel">tracelevel</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - Any positive number - </td> - </tr> - <tr><td>Default</td><td><em>No tracing</em></td></tr> - </table> - <p> - Set to a positive number to collect trace information for debugging - when running a query. Higher numbers give - progressively more detail on query transformations and searcher - execution. - </p> - - <h3 id="trace.timestamps">trace.timestamps</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td> - true or false - </td> - </tr> - <tr><td>Default</td><td><em>No timestamps in trace</em></td></tr> - </table> - <p> - Enable it to get timing information already at <a href="#tracelevel">tracelevel=1</a> which is useful for debugging latency spent at different components in the search chain without rendering a lot of string data which is associated with higher trace levels. - </p> - - - - <h2 id="query-model">Query Model Parameters</h2> - - <h3 id="model.defaultIndex">model.defaultIndex [<em>default-index</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>default-index</td></tr> - <tr><td>Values</td><td>An index name</td></tr> - <tr><td>Default</td><td><code>default</code></td></tr> - </table> - <p> - The field which is searched for query terms which doesn't explicitly specify an index. - </p> - - - <h3 id="model.encoding">model.encoding [<em>encoding</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>encoding</td></tr> - <tr><td>Values</td><td>Encoding names or aliases defined in the <a href="http://www.iana.org/assignments/character-sets">IANA character sets</a></td></tr> - <tr><td>Default</td><td>utf-8</td></tr> - </table> - <p> Sets the encoding to use when returning a result. The encodings <em>big5</em>, - <em>euc-jp</em>, <em>euc-kr</em>, <em>gb2312</em>, <em>iso-2022-jp</em> and <em>shift-jis</em> - also influences how tokenization is done in the absence of an explicit language setting. - </p><p> - The query is always encoded as UTF-8, independently of how the result will be encoded. - </p> - - - <h3 id="model.filter">model.filter [<em>filter</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>filter</td></tr> - <tr><td>Values</td><td>Any allowed collection of filter terms</td></tr> - <tr><td>Default</td><td><em>Not set</em></td></tr> - </table> - <p> - Sets a filter to be combined with the query. Typical use of a filter - is to add machine generated or preferences based filter terms to a raw - user query. The filter is parsed the same way as a query of type any, - the full syntax is available. The positive terms (preceded by +) and - phrases act as AND filters, the negative terms (preceded by -) act as - NOT filters, while the unprefixed terms will be used to RANK the - results. Unless the query has no positive terms, the filter will only - restrict and influence ranking of the result set, never cause more - matches than the query. - </p> - - - <h3 id="model.language">model.language [<em>lang, language</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>language, lang</td></tr> - <tr><td>Values</td><td>Ref. RFC 3066</td></tr> - <tr><td>Default</td><td></em>Unspecified</em></td></tr> - </table> - <p> - Informs Vespa about the natural language of the query. Please see - <a href="../linguistics.html">linguistics</a> for details. - This attribute should always be set when it is known. If this - parameter is not set, it will be guessed from the query and encoding, and - default to english if it cannot be guessed. - </p> - - - <h3 id="model.queryString">model.queryString [<em>query</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>query</td></tr> - <tr><td>Values</td><td>Any HTTP encoded legal Vespa query language string</td></tr> - <tr><td>Default</td><td><em>Not set</em></td></tr> - </table> - <p> - The <a href="simple-query-language-reference.html">Simple Vespa Query Language</a> query string - specifying which documents to match in this query. - </p> - - - <h3 id="model.restrict">model.restrict [<em>restrict</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>restrict</td></tr> - <tr><td>Values</td><td>A comma delimited list of document type names.</td></tr> - <tr><td>Default</td><td><em>Search unrestricted</em></td></tr> - </table> - <p> - The document types to restrict the search to when different document - types share the same search cluster. - </p> - - - <h3 id="model.searchPath">model.searchPath [<em>path</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>searchpath</td></tr> - <tr><td>Values</td><td><ul> - <li>searchpath::ELEMENT [';' ELEMENT]*</li> - <li>ELEMENT::PART ['/' ROW]</li> - <li>PART::EXP [',' EXP]*</li> - <li>EXP::NUM | RANGE</li> - <li>ROW::NUM</li> - <li>RANGE::'['NUM ',' NUM ' >'</li> - </ul></td></tr> - <tr><td>Default</td><td><em>Whole cluster</em></td></tr> - </table> - <p> - Specification of which path to send the query to. - Used to select which set of search nodes in the cluster should be used. - Only meant for debugging/monitoring. - </p><p> - Examples: - Note that in an indexed content cluster with flat distribution we have 1 implicit row - and each search node represents a part. - <ul> - <li>'7/3' = part 7, row 3.</li> - <li>'7/' = part 7, any row.</li> - <li>'7,1,9/0' = parts 1,7 and 9, row 0.</li> - <li>'1,[3,9>/0' = parts 1,3,4,5,6,7,8, row 0.</li> - </ul> - </p><p> - In a cluster with a multi-level dispatch setup we must specify a search path element for each level. - Lets say we have a setup with 2 mid-level dispatch groups, each containing 3 search nodes (and 3 dispatchers): - <ul> - <li>'0/;2/' = dispatch group (part) 0, any of the dispatchers (row); search node (part) 2, any row (of 1 present).</li> - <li>'0/1;2/0' = dispatch group (part) 0, dispatcher (row) 1; search node (part) 2, row 0 (of 1 present).</li> - </ul> - </p> - - - <h3 id="model.sources">model.sources [<em>search, sources</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>search, sources</td></tr> - <tr><td>Values</td><td>A comma separated list of search cluster names or other source names</td></tr> - <tr><td>Default</td><td><em>Search unrestricted</em></td></tr> - </table> - <p> - The names of the sources to search, e.g one or more search clusters and/or federated sources. - </p> - - - <h3 id="model.type">model.type [<em>type</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>type</td></tr> - <tr><td>Values</td><td>web, all, any, phrase, yql, adv (deprecated) - - refer to <a href="simple-query-language-reference.html">simple query language reference</a></td></tr> - <tr><td>Default</td><td>all</td></tr> - </table> - <p> - Selects the query language syntax of the <a href="#model.queryString">query</a> parameter. - </p> - - - - <h2 id="ranking">Ranking</h2> - - <h3 id="ranking.location">ranking.location [<em>location</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>location</td></tr> - <tr><td>Values</td><td>See <a href="../geo-search.html">Geo search</a></td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Point (one or two dimensional) location to use as base for location ranking. - For geographical locations, it is recommended to add the location using <a href="#pos.ll">pos.ll</a> - <!-- ToDo: Why? --> - </p> - - - <h3 id="ranking.features">ranking.features.<em>featurename</em> [<em>rankfeature.</em>featurename]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>rankfeature.featurename</td></tr> - <tr><td>Values</td><td>Any string</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Set a rank feature to a value. This works for any key name <code>query(anyname)</code> (query features), - and also as a way to override all existing (match and document) features. - Example: <em>query=foo&ranking.features.query(userage)=42&ranking.features.fieldMatch(title)=0.65</em> - </p> - - - <h3 id="ranking.listFeatures">ranking.listFeatures [<em>rankfeatures</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>rankfeatures</td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Set to true to request <em>all</em> rank features to be calculated and returned. - The rank features will be returned in the summary field rankfeatures. - This option is typically used for MLR training, should not to be used for production. - </p> - - - <h3 id="ranking.profile">ranking.profile [<em>ranking</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>ranking</td></tr> - <tr><td>Values</td><td>Any rank profile name</td></tr> - <tr><td>Default</td><td><code>default</code></td></tr> - </table> - <p> - Sets the name of the rank profile to use for assigning relevancy scores. - The default rank profile will be used for back-ends which does not have the given rank profile. - </p> - - - <h3 id="ranking.properties">ranking.properties.<em>propertyname</em> [<em>rankproperty.propertyname</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>rankproperty.propertyname</td></tr> - <tr><td>Values</td><td>Any string</td></tr> - <tr><td>Default</td><td><em>None</em></td></tr> - </table> - <p> - Set a rank property that is passed to, and used by a feature executor for this query. - Example: <em>query=foo&ranking.properties.dotProduct.X={a:1,b:2}</em> - </p> - - - <h3 id="ranking.sorting">ranking.sorting [<em>sorting</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>sorting</td></tr> - <tr><td>Values</td><td>A valid <a href="sorting.html">sort specification</a></td></tr> - <tr><td>Default</td><td>None - order by relevance</td></tr> - </table> - <p> - A specification of how to sort the result. - Fields you want to sort on must be stored as document attributes in the index structure - by adding <a href="search-definitions-reference.html#attribute">attribute</a> to the indexing statement. - </p> - - - <h3 id="ranking.freshness">ranking.freshness</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td><code>[integer]</code>, an absolute time in seconds since epoch, or <code>now-[number]</code>, to use a time [integer] seconds into the past, or <code>now</code> to use the current time</td></tr> - <tr><td>Default</td><td>None - use the current time on each node.</td></tr> - </table> - <p> - Sets the time which will be used as <em>now</em> during execution. - </p> - - - <h3 id="ranking.queryCache">ranking.queryCache</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Turns query cache on or off. Search is a two-phase process. If the - query cache is on, the query is stored on the search nodes between the - first and second phase, saving network bandwidth and also query setup - time, at the expense of using more memory. - </p> - - - <h3 id="ranking.matchPhase">ranking.matchPhase</h3> - <p>Settings which control Vespa's behavior during the match phase. - If these are set in the query they will override any match-phase setting - in the rank profile.</p> - <dt></dt> - <dd> - <ul> - <li><a href="#ranking.matchPhase.maxHits">ranking.matchPhase.maxHits</a> the max number of hits that should be generated during the match phase</li> - <li><a href="#ranking.matchPhase.attribute">ranking.matchPhase.attribute</a> the attribute to limit matches by if more than maxHits hits will be generated</li> - <li><a href="#ranking.matchPhase.ascending">ranking.matchPhase.ascending</a> whether to keep the documents having the highest (default) or lowest values of the attribute</li> - <li><a href="#ranking.matchPhase.diversity.attribute">ranking.matchPhase.diversity.attribute</a> the attribute to use to guarantee diversity.</li> - <li><a href="#ranking.matchPhase.diversity.minGroups">ranking.matchPhase.diversity.minGroups</a> the minimum number of groups grouped by the diversity attribute.</li> - </ul> - </dd> - - - <h3 id="ranking.matchPhase.maxHits">ranking.matchPhase.maxHits</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>long</td></tr> - <tr><td>Default</td><td>If sorting and not ranking: max(10000, maxhits+maxoffset). - Otherwise: <em>none</em>.</td></tr> - </table> - <p> - The max hits the engine should attempt to produce in the match phase on each partition. - If it is determined during matching that many more hits than this will be generated, the matching will fall back to - take the best (highest or lowest) values of the attribute given by ranking.matchPhase.attribute. - </p><p> - By default, this will be turned on only when sorting is used and grouping is not. - If sorting is used, the primary sort attribute will be used as the match phase attribute if it has fast-search set. - In that case the default can be overridden by setting this value explicitly. - </p> - - - <h3 id="ranking.matchPhase.attribute">ranking.matchPhase.attribute</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>An attribute name</td></tr> - <tr><td>Default</td><td><em>none</em></td></tr> - </table> - <p> - The attribute to decide which documents are a match if the match phase - estimates that there will be more than maxHits matches. - This attribute should have fast-search set and should correlate with the order - which would be produced by a full evaluation. - </p> - - - <h3 id="ranking.matchPhase.ascending">ranking.matchPhase.ascending</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Whether the attribute should be sorted in ascending or descending (default) order - to determine which documents to keep as matches. - </p> - - - <h3 id="ranking.matchPhase.diversity.attribute">ranking.matchPhase.diversity.attribute</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>An attribute name</td></tr> - <tr><td>Default</td><td>none.</td></tr> - </table> - <p> - The attribute to be used for producing the desired diversity. - Also see <a href="search-definitions-reference.html#diversity-attribute">attribute</a>. - </p> - - - <h3 id="ranking.matchPhase.diversity.minGroups">ranking.matchPhase.diversity.minGroups</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>long</td></tr> - <tr><td>Default</td><td>none</td></tr> - </table> - <p> - The minimum number of groups that should be returned from the match phase grouped by the diversity attribute. - Also see <a href="search-definitions-reference.html#diversity-min-groups">min-groups</a>. - </p> - - - - <h2 id="presentation">Presentation</h2> - - <h3 id="presentation.bolding">presentation.bolding [<em>bolding</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>bolding</td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>true</td></tr> - </table> - <p> - Whether or not to bold search terms in <a href="search-definitions-reference.html">search definition</a> - fields defined with <a href="search-definitions-reference.html#bolding">bolding: on</a> - or <a href="search-definitions-reference.html#summary">summary: dynamic</a>. - </p> - - - <h3 id="presentation.format">presentation.format [<em>format</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>format</td></tr> - <tr><td>Values</td><td> - <table class="table table-striped"> - <tr> - <td><em>No value</em> or <code><a href="default-result-format.html">default</a></code></td> - <td>The default, builtin JSON format</td> - </tr> - <tr> - <td><code><a href="default-result-format.html">json</a></code></td> - <td>Builtin JSON format</td> - </tr> - <tr> - <td><code>xml</code></td> - <td>Deprecated, builtin XML format</td> - </tr> - <tr> - <td><code><a href="page-result-format.html">page</a></code></td> - <td>Alternative deprecated XML format which is suitable for use with <a href="../page-templates.html">page templates</a>.</td> - </tr> - <tr> - <td><em>Any other value</em></td> - <td>A custom <a href="../result-rendering.html">result renderer</a> supplied by the application - </tr> - </table> - - </td></tr> - <tr><td>Default</td><td>default</td></tr> - </table> - - - <h3 id="presentation.summary">presentation.summary [<em>summary</em>]</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td>summary</td></tr> - <tr><td>Values</td><td> - The name of the <a href="../document-summaries.html#summary-classes-in-queries">summary class</a> - used to select fields in results. - </td></tr> - <tr><td>Default</td><td>The default summary class of the search definition.</td></tr> - </table> - - - <h3 id="presentation.template">presentation.template</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Any id specification of a deployed page template.</td></tr> - <tr><td>Default</td><td></td></tr> - </table> - <p> - The id of the page template to use for this result. This should be used with the - <a href="page-result-format.html">page</a> result format. - </p> - - - <h3 id="presentation.timing">presentation.timing</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>boolean</td></tr> - <tr><td>Default</td><td>false</td></tr> - </table> - <p> - Whether a result renderer should try to add optional timing information - to the rendered page. - </p> - - - - <h2 id="">Grouping and Aggregation</h2> - - <h3 id="select">select</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A valid grouping specification.</td></tr> - <tr><td>Default</td><td>No grouping</td></tr> - </table> - <p> - Requests specific multi-level result set statistics and/or hit groups to be returned in the result. - Fields you want to retrieve statistics or hit groups for must be stored as document attributes - in the index structure by adding attribute to the indexing statement. - See the <a href="../grouping.html">grouping guide</a>. - </p> - - - <h3 id="collapsefield">collapsefield</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Any document summary field name</td></tr> - <tr><td>Default</td><td>No field collapsing</td></tr> - </table> - <p> - Collapse (i.e. aggregate) results using this field. - Collapsing is run in the container, not content node level. - Define a <em>collapsefield</em> to remove duplicates if the corpus has few duplicates - - this is more efficient than using <a href="#select">grouping</a>. - Otherwise, use grouping. - </p> - - - <h3 id="collapsesize">collapsesize</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A positive integer</td></tr> - <tr><td>Default</td><td>1</td></tr> - </table> - <p>The number of hits to keep in each collapsed bucket</p> - - - <h3 id="collapse.summary">collapse.summary</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A valid name of a document summary class.</td></tr> - <tr><td>Default</td><td>Use default summary or attributes.</td></tr> - </table> - <p>Use this summary class to fetch the field used for collapsing.</p> - - - - <h2 id="geographical-searches">Geographical Searches</h2> - - <h3 id="pos.ll">pos.ll</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td> - <td>Position given in latitude and longitude - example: <em>S22.4532;W123.9887</em> - Refer to <a href="search-definitions-reference.html#type:position">position field</a> - for format specification.</td> - </tr> - <tr><td>Default</td><td>None</td></tr> - </table> - - - <h3 id="pos.radius">pos.radius</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td> - Radius of the circle used for filtering. Valid units of measurement are km, m and mi. Examples: - <ul> - <li>pos.radius=100m</li> - <li>pos.radius=42mi</li> - <li>pos.radius=4km</li> - </ul> - One can also specify just a number (internal units, micro-degrees), but this is not recommended. - </td></tr> - <tr><td>Default</td><td>50km</td></tr> - </table> - - - <h3 id="pos.bb">pos.bb</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td> - Bounding box for positions, given as latitude and longitude boundaries. - The four boundaries must be specified as N, S, E, W, with degrees as - a decimal fraction. Degrees south of equator or west of Greenwich are - input as negative numbers. Examples: - <ul> - <li>n=37.44899,s=37.3323,e=-121.98241,w=-122.06566</li> - <li>s=40.183868,w=-74.819519,n=40.248291,e=-74.728798</li> - </ul> - </td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - - - <h3 id="pos.attribute">pos.attribute</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Any attribute that has zcurve encoded positions as a long attribute.</td></tr> - <tr><td>Default</td><td>Random choice among the ones declared as position in the searchdefinition.</td></tr> - </table> - <p> - Which attribute to use for the position. Can be both single- or multi-value. - </p> - - - - <h2 id="">Streaming Search</h2> - <p> - The features in this section applies to <a href="../streaming-search.html">streaming search</a> only. - </p> - - <h3 id="streaming.userid">streaming.userid</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>An integer in decimal notation in the range [0, 2^64></td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Restricts streaming search to only stream through documents with document ids having the n=<number> - modifier and the userid part matches the supplied value. This can be used for grouping documents on a 64 bit integer. - </p> - - - <h3 id="streaming.groupname">streaming.groupname</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A string</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Restricts streaming search to only stream through documents with document ids having the g=<groupname> - modifier and the groupname part matches the supplied value. This can be used for grouping documents on a string. - </p> - - - <h3 id="streaming.selection">streaming.selection</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A string</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - Restricts streaming search using a <a href="document-select-language.html">document selection</a>. - This can be used for selecting a subset of documents based on an advanced expression. - </p> - - - <h3 id="streaming.priority">streaming.priority</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td><a href="services.html#load-types">Priority class</a></td></tr> - <tr><td>Default</td><td>VERY_HIGH</td></tr> - </table> - <p> - Priority of the streaming search visitor. Having a high priority visitor helps maintain low latencies - even when the system is under load. - </p> - - - <h3 id="streaming.maxbucketspervisitor">streaming.maxbucketspervisitor</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>int</td></tr> - <tr><td>Default</td><td>1 (if ordering is set), or infinite</td></tr> - </table> - <p> - If set, visit only this many buckets at a time. - Combine with ordering to reduce visiting time for large users/groups. - </p> - - - - <h2 id="semantic-rules">Semantic Rules</h2> - <p> - Refer to <a href="semantic-rules.html">semantic rules</a>. - </p> - - <h3 id="rules.off">rules.off</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Boolean</td></tr> - <tr><td>Default</td><td>True</td></tr> - </table> - <p>Turn rule evaluation off for this query</p> - - - <h3 id="rules.rulebase">rules.rulebase</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>String</td></tr> - <tr><td>Default</td><td>A rule base name</td></tr> - </table> - <p>The name of the rule base to use for these queries</p> - - - <h3 id="tracelevel.rules">tracelevel.rules</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>int</td></tr> - <tr><td>Default</td><td>1-5 (?)</td></tr> - </table> - <p> - The amount of rule evaluation trace output to show, higher number means more details. - This is useful to see a trace from rule evaluation - without having to see trace from all other searchers at the same time. - </p> - - - - <h2 id="other">Other</h2> - - <h3 id="recall">recall</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Any allowed collection of recall terms</td></tr> - <tr><td>Default</td><td>No recall</td></tr> - </table> - <p> - Sets a recall parameter to be combined with the query. - This is identical to <a href="#model.filter">filter</a>, - except that recall terms are not exposed to the ranking framework and thus not ranked. - As such, one can not use unprefixed terms; they must either by positive or negative. - </p> - - - <h3 id="user">user</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>A string</td></tr> - <tr><td>Default</td><td>None</td></tr> - </table> - <p> - The id of the user making the query. The contents of the argument are made available to the search chain, - but it triggers no features in Vespa apart from being propagated to the access log. - </p> - - - <h3 id="nocachewrite">nocachewrite</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Boolean</td></tr> - <tr><td>Default</td><td>False</td></tr> - </table> - <p>Set to true to avoid the result being written to cache when fetched.</p> - - - <h3 id="hitcountestimate">hitcountestimate</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Boolean</td></tr> - <tr><td>Default</td><td>False</td></tr> - </table> - <p>Make this an estimation query. - No hits will be returned, and total hit count will be set to an estimate of what executing - the query as a normal query would give. - </p> - - <h3 id="metrics.ignore">metrics.ignore</h3> - <table class="table table-striped"> - <tr><td>Alias</td><td></td></tr> - <tr><td>Values</td><td>Boolean</td></tr> - <tr><td>Default</td><td>False</td></tr> - </table> - <p>Ignore metric collection for this query request, useful for warm up queries</p> - - - </div> - - </div> -</div> - -</body> -</html> diff --git a/container-search-gui/src/main/resources/gui/icons/android-chrome-192x192.png b/container-search-gui/src/main/resources/gui/icons/android-chrome-192x192.png Binary files differindex af80e645b77..2e2d8952f86 100644 --- a/container-search-gui/src/main/resources/gui/icons/android-chrome-192x192.png +++ b/container-search-gui/src/main/resources/gui/icons/android-chrome-192x192.png diff --git a/container-search-gui/src/main/resources/gui/icons/android-chrome-512x512.png b/container-search-gui/src/main/resources/gui/icons/android-chrome-512x512.png Binary files differnew file mode 100644 index 00000000000..14724b65b30 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/android-chrome-512x512.png diff --git a/container-search-gui/src/main/resources/gui/icons/apple-touch-icon.png b/container-search-gui/src/main/resources/gui/icons/apple-touch-icon.png Binary files differindex 4dfc9a85a2a..402b914a17a 100644 --- a/container-search-gui/src/main/resources/gui/icons/apple-touch-icon.png +++ b/container-search-gui/src/main/resources/gui/icons/apple-touch-icon.png diff --git a/container-search-gui/src/main/resources/gui/icons/browserconfig.xml b/container-search-gui/src/main/resources/gui/icons/browserconfig.xml index efa70aac49b..6a1d00efa0e 100644 --- a/container-search-gui/src/main/resources/gui/icons/browserconfig.xml +++ b/container-search-gui/src/main/resources/gui/icons/browserconfig.xml @@ -1,10 +1,9 @@ <?xml version="1.0" encoding="utf-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <browserconfig> <msapplication> <tile> <square150x150logo src="/icons/mstile-150x150.png"/> - <TileColor>#2b5797</TileColor> + <TileColor>#da532c</TileColor> </tile> </msapplication> </browserconfig> diff --git a/container-search-gui/src/main/resources/gui/icons/favicon-16x16.png b/container-search-gui/src/main/resources/gui/icons/favicon-16x16.png Binary files differindex cce56472694..5cc83a38dba 100644 --- a/container-search-gui/src/main/resources/gui/icons/favicon-16x16.png +++ b/container-search-gui/src/main/resources/gui/icons/favicon-16x16.png diff --git a/container-search-gui/src/main/resources/gui/icons/favicon-32x32.png b/container-search-gui/src/main/resources/gui/icons/favicon-32x32.png Binary files differindex 5f9f3e1b85b..dfe72c4aa0a 100644 --- a/container-search-gui/src/main/resources/gui/icons/favicon-32x32.png +++ b/container-search-gui/src/main/resources/gui/icons/favicon-32x32.png diff --git a/container-search-gui/src/main/resources/gui/icons/favicon.ico b/container-search-gui/src/main/resources/gui/icons/favicon.ico Binary files differindex 885fb77541d..52564baa44d 100644 --- a/container-search-gui/src/main/resources/gui/icons/favicon.ico +++ b/container-search-gui/src/main/resources/gui/icons/favicon.ico diff --git a/container-search-gui/src/main/resources/gui/icons/manifest.json b/container-search-gui/src/main/resources/gui/icons/manifest.json index f71a0007cf4..5b2f3618461 100644 --- a/container-search-gui/src/main/resources/gui/icons/manifest.json +++ b/container-search-gui/src/main/resources/gui/icons/manifest.json @@ -1,5 +1,6 @@ { "name": "", + "short_name": "", "icons": [ { "src": "/icons/android-chrome-192x192.png", @@ -7,12 +8,12 @@ "type": "image/png" }, { - "src": "/icons/android-chrome-384x384.png", - "sizes": "384x384", + "src": "/icons/android-chrome-512x512.png", + "sizes": "512x512", "type": "image/png" } ], "theme_color": "#ffffff", "background_color": "#ffffff", "display": "standalone" -}
\ No newline at end of file +} diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-144x144.png b/container-search-gui/src/main/resources/gui/icons/mstile-144x144.png Binary files differnew file mode 100644 index 00000000000..9eacb6051d8 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/mstile-144x144.png diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-150x150.png b/container-search-gui/src/main/resources/gui/icons/mstile-150x150.png Binary files differindex 15ba5c1404e..736a5ca47d4 100644 --- a/container-search-gui/src/main/resources/gui/icons/mstile-150x150.png +++ b/container-search-gui/src/main/resources/gui/icons/mstile-150x150.png diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-310x150.png b/container-search-gui/src/main/resources/gui/icons/mstile-310x150.png Binary files differnew file mode 100644 index 00000000000..0542b761ba4 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/mstile-310x150.png diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-310x310.png b/container-search-gui/src/main/resources/gui/icons/mstile-310x310.png Binary files differnew file mode 100644 index 00000000000..5fc7990af48 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/mstile-310x310.png diff --git a/container-search-gui/src/main/resources/gui/icons/mstile-70x70.png b/container-search-gui/src/main/resources/gui/icons/mstile-70x70.png Binary files differnew file mode 100644 index 00000000000..4064cb227c4 --- /dev/null +++ b/container-search-gui/src/main/resources/gui/icons/mstile-70x70.png diff --git a/container-search-gui/src/main/resources/gui/icons/safari-pinned-tab.svg b/container-search-gui/src/main/resources/gui/icons/safari-pinned-tab.svg index 7b775fcdbde..5589b2f5ed6 100644 --- a/container-search-gui/src/main/resources/gui/icons/safari-pinned-tab.svg +++ b/container-search-gui/src/main/resources/gui/icons/safari-pinned-tab.svg @@ -2,21 +2,23 @@ <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd"> <svg version="1.0" xmlns="http://www.w3.org/2000/svg" - width="464.000000pt" height="464.000000pt" viewBox="0 0 464.000000 464.000000" + width="700.000000pt" height="700.000000pt" viewBox="0 0 700.000000 700.000000" preserveAspectRatio="xMidYMid meet"> <metadata> -Created by potrace 1.11, written by Peter Selinger 2001-2013 +Created by potrace 1.14, written by Peter Selinger 2001-2017 </metadata> -<g transform="translate(0.000000,464.000000) scale(0.100000,-0.100000)" +<g transform="translate(0.000000,700.000000) scale(0.100000,-0.100000)" fill="#000000" stroke="none"> -<path d="M2559 4542 c-129 -88 -531 -360 -599 -406 -25 -17 -49 -34 -55 -38 --5 -5 -26 -19 -45 -31 -35 -23 -183 -124 -761 -517 -68 -46 -144 -97 -169 --114 -25 -16 -46 -31 -48 -32 -2 -1 -4 -126 -4 -278 l1 -276 -27 18 c-15 11 --148 104 -297 209 -148 104 -294 206 -323 226 l-52 37 0 -1052 1 -1053 627 --425 c919 -624 937 -636 1056 -717 l109 -75 196 133 c603 409 710 482 1061 -721 146 99 327 221 402 271 l137 92 3 277 3 277 285 -201 c157 -110 313 -220 -348 -245 l62 -44 0 1053 -1 1053 -92 62 c-50 34 -144 97 -207 140 -63 43 -214 -146 -335 228 -121 82 -431 292 -688 467 -258 175 -469 318 -471 318 -1 0 -54 --35 -117 -78z"/> +<path d="M3820 6438 c-74 -50 -189 -128 -255 -173 -66 -45 -203 -138 -305 +-207 -325 -220 -883 -599 -1220 -828 -77 -52 -209 -142 -295 -200 l-155 -105 +-2 -363 -3 -362 -454 320 c-250 176 -455 320 -457 320 -2 0 -4 -623 -4 -1384 +l0 -1384 262 -178 c145 -98 304 -205 353 -239 50 -34 169 -115 265 -180 96 +-65 216 -146 265 -180 84 -57 163 -111 645 -438 102 -70 271 -185 377 -257 +105 -71 194 -130 198 -130 6 0 161 104 530 355 66 45 212 145 325 221 113 76 +315 213 448 304 133 91 322 219 420 285 97 66 247 168 332 226 85 58 191 130 +235 160 l80 54 0 362 c0 200 1 363 3 363 4 0 45 -29 503 -352 207 -147 386 +-271 398 -277 l21 -12 0 1384 0 1383 -247 168 c-137 92 -369 250 -518 351 +-148 101 -394 267 -545 370 -151 103 -358 243 -460 312 -102 69 -268 182 -370 +251 -102 69 -192 131 -201 139 -8 7 -20 13 -25 13 -5 0 -70 -41 -144 -92z"/> </g> </svg> diff --git a/container-search-gui/src/main/resources/gui/img/Vespa-V2.png b/container-search-gui/src/main/resources/gui/img/Vespa-V2.png Binary files differdeleted file mode 100644 index ac87f8e94d0..00000000000 --- a/container-search-gui/src/main/resources/gui/img/Vespa-V2.png +++ /dev/null diff --git a/container-search-gui/src/main/resources/gui/img/VespaIcon.png b/container-search-gui/src/main/resources/gui/img/VespaIcon.png Binary files differdeleted file mode 100644 index 33063432c20..00000000000 --- a/container-search-gui/src/main/resources/gui/img/VespaIcon.png +++ /dev/null diff --git a/container-search-gui/src/main/resources/gui/img/information.svg b/container-search-gui/src/main/resources/gui/img/information.svg deleted file mode 100644 index da42cf2caf6..00000000000 --- a/container-search-gui/src/main/resources/gui/img/information.svg +++ /dev/null @@ -1,10 +0,0 @@ -<?xml version="1.0"?> -<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Capa_1" x="0px" y="0px" viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve" width="512px" height="512px"><g><g> - <g> - <g> - <circle cx="256" cy="378.5" r="25" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> - <path d="M256,0C114.516,0,0,114.497,0,256c0,141.484,114.497,256,256,256c141.484,0,256-114.497,256-256 C512,114.516,397.503,0,256,0z M256,472c-119.377,0-216-96.607-216-216c0-119.377,96.607-216,216-216 c119.377,0,216,96.607,216,216C472,375.377,375.393,472,256,472z" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> - <path d="M256,128.5c-44.112,0-80,35.888-80,80c0,11.046,8.954,20,20,20s20-8.954,20-20c0-22.056,17.944-40,40-40 c22.056,0,40,17.944,40,40c0,22.056-17.944,40-40,40c-11.046,0-20,8.954-20,20v50c0,11.046,8.954,20,20,20 c11.046,0,20-8.954,20-20v-32.531c34.466-8.903,60-40.26,60-77.469C336,164.388,300.112,128.5,256,128.5z" data-original="#000000" class="active-path" data-old_color="#898989" fill="#767474"/> - </g> - </g> -</g></g> </svg> diff --git a/container-search-gui/src/main/resources/gui/img/reload.svg b/container-search-gui/src/main/resources/gui/img/reload.svg deleted file mode 100644 index c5381f9f232..00000000000 --- a/container-search-gui/src/main/resources/gui/img/reload.svg +++ /dev/null @@ -1,6 +0,0 @@ -<?xml version="1.0"?> -<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Layer_1" x="0px" y="0px" viewBox="0 0 512 512" style="enable-background:new 0 0 512 512;" xml:space="preserve" width="512px" height="512px" class=""><g><g> - <g> - <path d="M482.195,226.196C482.195,101.471,380.725,0,256.001,0S29.805,101.471,29.805,226.196c0,7.409,6.007,13.416,13.416,13.416 s13.416-6.008,13.416-13.416c0-109.93,89.434-199.363,199.363-199.363s199.363,89.434,199.363,199.363 c0,109.928-89.434,199.362-199.363,199.362h-23.276l33.282-37.255c4.937-5.525,4.458-14.007-1.067-18.944 c-5.525-4.937-14.008-4.457-18.944,1.068l-47.576,53.255c-7.788,8.718-7.788,21.866,0,30.584l47.576,53.255 c2.651,2.968,6.322,4.478,10.01,4.478c3.181,0,6.375-1.126,8.934-3.41c5.526-4.937,6.004-13.419,1.067-18.944l-33.282-37.255 h23.276C380.725,452.39,482.195,350.919,482.195,226.196z" data-original="#000000" class="active-path" data-old_color="#F0EDED" fill="#F3F2F2"/> - </g> -</g></g> </svg> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 8546eb5a971..281ac50e63a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Instant; @@ -88,15 +87,14 @@ public class ApplicationPackageValidator { /** Verify that each of the production zones listed in the deployment spec exist in this system */ private void validateSteps(DeploymentSpec deploymentSpec) { for (var spec : deploymentSpec.instances()) { - new DeploymentSteps(spec, controller.zoneRegistry()).jobs(); - spec.zones().stream() - .filter(zone -> zone.environment() == Environment.prod) - .forEach(zone -> { - if ( ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), - zone.region().orElseThrow()))) { - throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); - } - }); + for (var zone : spec.zones()) { + if (zone.environment().isManuallyDeployed()) + throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); + + if ( zone.environment() == Environment.prod + && ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), zone.region().orElseThrow()))) + throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index f0d172eeac6..7578c133fc6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -14,6 +14,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.stream.CustomCollectors; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -76,6 +77,7 @@ public class DeploymentStatus { private final VersionStatus versionStatus; private final Version systemVersion; private final Function<InstanceName, VersionCompatibility> versionCompatibility; + private final ZoneRegistry zones; private final Instant now; private final Map<JobId, StepStatus> jobSteps; private final List<StepStatus> allSteps; @@ -83,6 +85,7 @@ public class DeploymentStatus { public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones, VersionStatus versionStatus, Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) { this.application = requireNonNull(application); + this.zones = zones; this.systemTest = JobType.systemTest(zones); this.stagingTest = JobType.stagingTest(zones); this.versionStatus = requireNonNull(versionStatus); @@ -148,8 +151,7 @@ public class DeploymentStatus { public Map<JobType, JobStatus> instanceJobs(InstanceName instance) { return allJobs.asList().stream() .filter(job -> job.id().application().equals(application.id().instance(instance))) - .collect(Collectors.toUnmodifiableMap(job -> job.id().type(), - Function.identity())); + .collect(CustomCollectors.toLinkedMap(job -> job.id().type(), Function.identity())); } /** Filterable job status lists for each instance of this application. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java deleted file mode 100644 index 44079a90097..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.application.Deployment; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Comparator.comparingInt; -import static java.util.stream.Collectors.collectingAndThen; - -/** - * This class provides helper methods for reading a deployment spec. - * - * @author mpolden - */ -public class DeploymentSteps { - - private final DeploymentInstanceSpec spec; - private final ZoneRegistry zones; - - public DeploymentSteps(DeploymentInstanceSpec spec, ZoneRegistry zones) { - this.spec = Objects.requireNonNull(spec, "spec cannot be null"); - this.zones = Objects.requireNonNull(zones, "system cannot be null"); - } - - /** Returns jobs for this, in the order they should run */ - public List<JobType> jobs() { - return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest(zones), JobType.stagingTest(zones)), - spec.steps().stream().flatMap(step -> toJobs(step).stream())) - .distinct() - .collect(Collectors.toUnmodifiableList()); - } - - /** Returns job status sorted according to deployment spec */ - public List<JobStatus> sortedJobs(Collection<JobStatus> jobStatus) { - List<JobType> sortedJobs = jobs(); - return jobStatus.stream() - .sorted(comparingInt(job -> sortedJobs.indexOf(job.id().type()))) - .collect(Collectors.toUnmodifiableList()); - } - - /** Returns deployments sorted according to declared zones */ - public List<Deployment> sortedDeployments(Collection<Deployment> deployments) { - List<ZoneId> productionZones = spec.zones().stream() - .filter(z -> z.region().isPresent()) - .map(z -> ZoneId.from(z.environment(), z.region().get())) - .collect(Collectors.toUnmodifiableList()); - return deployments.stream() - .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) - .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); - } - - /** Resolve jobs from step */ - public List<JobType> toJobs(DeploymentSpec.Step step) { - return step.zones().stream() - .map(this::toJob) - .collect(Collectors.toUnmodifiableList()); - } - - /** Returns test jobs to run for this spec */ - public List<JobType> testJobs() { - return jobs().stream().filter(type -> type.environment().isTest()).collect(Collectors.toUnmodifiableList()); - } - - /** Returns declared production jobs in this */ - public List<JobType> productionJobs() { - return toJobs(production()); - } - - /** Returns declared production steps in this */ - public List<DeploymentSpec.Step> production() { - return spec.steps().stream() - .filter(step -> ! isTest(step)) - .collect(Collectors.toUnmodifiableList()); - } - - private boolean isTest(DeploymentSpec.Step step) { - return step.concerns(Environment.test) || step.concerns(Environment.staging); - } - - /** Resolve job from deployment zone */ - private JobType toJob(DeploymentSpec.DeclaredZone zone) { - switch (zone.environment()) { - case prod: return JobType.prod(zone.region().get()); - case test: return JobType.systemTest(zones); - case staging: return JobType.stagingTest(zones); - default: throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); - } - } - - /** Resolve jobs from steps */ - private List<JobType> toJobs(List<DeploymentSpec.Step> steps) { - return steps.stream() - .flatMap(step -> toJobs(step).stream()) - .collect(Collectors.toUnmodifiableList()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 419d6155fe7..df8ee910dc4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -71,10 +71,6 @@ public class DeploymentTrigger { this.jobs = controller.jobController(); } - public DeploymentSteps steps(DeploymentInstanceSpec spec) { - return new DeploymentSteps(spec, controller.zoneRegistry()); - } - /** * Propagates the latest revision to ready instances. * Ready instances are those whose dependencies are complete, and which aren't blocked, and, additionally, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 35eaa36cd2f..5113d386b23 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableSortedMap; import com.yahoo.component.Version; import com.yahoo.component.VersionCompatibility; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -41,6 +42,7 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Deque; @@ -627,7 +629,7 @@ public class JobController { DeploymentId deploymentId = new DeploymentId(id, type.zone()); Optional<Run> lastRun = last(id, type); - lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); + lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id(), Duration.ofMinutes(2))); long build = 1 + lastRun.map(run -> run.versions().targetRevision().number()).orElse(0L); RevisionId revisionId = RevisionId.forDevelopment(build, new JobId(id, type)); @@ -668,14 +670,6 @@ public class JobController { } private Version findTargetPlatform(ApplicationPackage applicationPackage, DeploymentId id, Optional<Instance> instance) { - Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); - if (major.isPresent()) - return controller.applications().lastCompatibleVersion(major.get()) - .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " + - "but no version on this major was found")); - - VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); - // Prefer previous platform if possible. Candidates are all deployable, ascending, with existing version appended; then reversed. List<Version> versions = controller.readVersionStatus().deployableVersions().stream() .map(VespaVersion::versionNumber) @@ -685,24 +679,47 @@ public class JobController { .map(Deployment::version) .ifPresent(versions::add); + if (versions.isEmpty()) + throw new IllegalStateException("no deployable platform version found in the system"); + + VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); + List<Version> compatibleVersions = new ArrayList<>(); for (Version target : reversed(versions)) if (applicationPackage.compileVersion().isEmpty() || compatibility.accept(target, applicationPackage.compileVersion().get())) + compatibleVersions.add(target); + + if (compatibleVersions.isEmpty()) + throw new IllegalArgumentException("no platforms are compatible with compile version " + applicationPackage.compileVersion().get()); + + Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); + List<Version> versionOnRightMajor = new ArrayList<>(); + for (Version target : reversed(versions)) + if (major.isEmpty() || major.get() == target.getMajor()) + versionOnRightMajor.add(target); + + if (versionOnRightMajor.isEmpty()) + throw new IllegalArgumentException("no platforms were found for major version " + major.get() + " specified in deployment.xml"); + + for (Version target : compatibleVersions) + if (versionOnRightMajor.contains(target)) return target; - throw new IllegalArgumentException("no suitable platform version found" + - applicationPackage.compileVersion() - .map(version -> " for package compiled against " + version) - .orElse("")); + throw new IllegalArgumentException("no platforms on major version " + major.get() + " specified in deployment.xml " + + "are compatible with compile version " + applicationPackage.compileVersion().get()); } /** Aborts a run and waits for it complete. */ - private void abortAndWait(RunId id) { + private void abortAndWait(RunId id, Duration timeout) { abort(id, "replaced by new deployment"); runner.get().accept(last(id.application(), id.type()).get()); + Instant doom = controller.clock().instant().plus(timeout); + Duration sleep = Duration.ofMillis(100); while ( ! last(id.application(), id.type()).get().hasEnded()) { + if (controller.clock().instant().plus(sleep).isAfter(doom)) + throw new UncheckedTimeoutException("timeout waiting for " + id + " to abort and finish"); try { - Thread.sleep(100); + Thread.sleep(sleep.toMillis()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index 5178918aa48..974345330aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -20,7 +20,8 @@ public class ReadyJobsTrigger extends ControllerMaintainer { @Override public double maintain() { TriggerResult result = controller().applications().deploymentTrigger().triggerReadyJobs(); - return result.triggered() * 1.0f / (result.triggered() + result.failed()); + long total = result.triggered() + result.failed(); + return total == 0 ? 1 : (double) result.triggered() / (result.triggered() + result.failed()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 63f33540721..cf21b8ef0af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -12,6 +12,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import com.yahoo.component.annotation.Inject; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -90,7 +91,6 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; @@ -141,6 +141,7 @@ import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Base64; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -158,7 +159,9 @@ import java.util.stream.Stream; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; import static com.yahoo.yolean.Exceptions.uncheck; +import static java.util.Comparator.comparingInt; import static java.util.Map.Entry.comparingByKey; +import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toUnmodifiableList; @@ -1528,10 +1531,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setString("instance", instance.name().value()); if (deploymentSpec.instance(instance.name()).isPresent()) { - // Jobs sorted according to deployment spec - List<JobStatus> jobStatus = controller.applications().deploymentTrigger() - .steps(deploymentSpec.requireInstance(instance.name())) - .sortedJobs(status.instanceJobs(instance.name()).values()); + // Jobs ordered according to deployment spec + Collection<JobStatus> jobStatus = status.instanceJobs(instance.name()).values(); if ( ! instance.change().isEmpty()) toSlime(object.setObject("deploying"), instance.change(), status.application()); @@ -1559,8 +1560,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = deploymentSpec.instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) - .map(steps -> steps.sortedDeployments(instance.deployments().values())) + .map(spec -> sortedDeployments(instance.deployments().values(), spec)) .orElse(List.copyOf(instance.deployments().values())); Cursor deploymentsArray = object.setArray("deployments"); @@ -1613,10 +1613,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { application.projectId().ifPresent(id -> object.setLong("projectId", id)); if (application.deploymentSpec().instance(instance.name()).isPresent()) { - // Jobs sorted according to deployment spec - List<JobStatus> jobStatus = controller.applications().deploymentTrigger() - .steps(application.deploymentSpec().requireInstance(instance.name())) - .sortedJobs(status.instanceJobs(instance.name()).values()); + // Jobs ordered according to deployment spec + Collection<JobStatus> jobStatus = status.instanceJobs(instance.name()).values(); if ( ! instance.change().isEmpty()) toSlime(object.setObject("deploying"), instance.change(), application); @@ -1645,11 +1643,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { addRotationId(object, instance); // Deployments sorted according to deployment spec - List<Deployment> deployments = - application.deploymentSpec().instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) - .map(steps -> steps.sortedDeployments(instance.deployments().values())) - .orElse(List.copyOf(instance.deployments().values())); + List<Deployment> deployments = application.deploymentSpec().instance(instance.name()) + .map(spec -> sortedDeployments(instance.deployments().values(), spec)) + .orElse(List.copyOf(instance.deployments().values())); Cursor instancesArray = object.setArray("instances"); for (Deployment deployment : deployments) { Cursor deploymentObject = instancesArray.addObject(); @@ -3017,5 +3013,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } } + private List<Deployment> sortedDeployments(Collection<Deployment> deployments, DeploymentInstanceSpec spec) { + List<ZoneId> productionZones = spec.zones().stream() + .filter(z -> z.region().isPresent()) + .map(z -> ZoneId.from(z.environment(), z.region().get())) + .toList(); + return deployments.stream() + .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) + .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index b80a487ec50..44a8b636ae0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -91,16 +92,14 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); var tenant = tenants.require(tenantName, CloudTenant.class); - var plan = planRegistry.plan(billing.getPlan(tenant.name())).orElseThrow(); + var plan = planFor(tenant.name()); var collectionMethod = billing.getCollectionMethod(tenant.name()); var response = new Slime(); var cursor = response.setObject(); cursor.setString("tenant", tenant.name().value()); - var planCursor = cursor.setObject("plan"); - planCursor.setString("id", plan.id().value()); - planCursor.setString("name", plan.displayName()); + toSlime(cursor.setObject("plan"), plan); cursor.setString("collection", collectionMethod.name()); return response; } @@ -137,7 +136,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var response = new Slime(); var cursor = response.setObject(); cursor.setString("tenant", tenant.name().value()); - cursor.setString("plan", billing.getPlan(tenant.name()).value()); + toSlime(cursor.setObject("plan"), planFor(tenant.name())); cursor.setString("collection", billing.getCollectionMethod(tenant.name()).name()); return response; } @@ -205,7 +204,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var usage = Optional.ofNullable(usagePerTenant.get(tenant.name())); var tenantResponse = tenantsResponse.addObject(); tenantResponse.setString("tenant", tenant.name().value()); - tenantResponse.setString("plan", billing.getPlan(tenant.name()).value()); + toSlime(tenantResponse.setObject("plan"), planFor(tenant.name())); tenantResponse.setString("collection", billing.getCollectionMethod(tenant.name()).name()); tenantResponse.setString("lastBill", usage.map(Bill::getStartDate).map(DateTimeFormatter.ISO_DATE::format).orElse(null)); tenantResponse.setString("unbilled", usage.map(Bill::sum).map(BigDecimal::toPlainString).orElse("0.00")); @@ -304,8 +303,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler slime.setString("id", item.id()); slime.setString("description", item.description()); slime.setString("amount",item.amount().toString()); - slime.setString("plan", item.plan()); - slime.setString("planName", billing.getPlanDisplayName(PlanId.from(item.plan()))); + toSlime(slime.setObject("plan"), planRegistry.plan(item.plan()).orElseThrow(() -> new RuntimeException("No such plan: '" + item.plan() + "'"))); item.applicationId().ifPresent(appId -> { slime.setString("application", appId.application().value()); @@ -354,4 +352,14 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return inspector.field(field).asString(); } + private void toSlime(Cursor cursor, Plan plan) { + cursor.setString("id", plan.id().value()); + cursor.setString("name", plan.displayName()); + } + + private Plan planFor(TenantName tenant) { + var planId = billing.getPlan(tenant); + return planRegistry.plan(planId) + .orElseThrow(() -> new RuntimeException("No such plan: '" + planId + "'")); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 8ce55fa5f1b..28536f36e20 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -748,15 +748,15 @@ public class ControllerTest { fail("Should fail when specifying a major that does not yet exist"); } catch (IllegalArgumentException e) { - assertEquals("major 8 specified in deployment.xml, but no version on this major was found", e.getMessage()); + assertEquals("no platforms were found for major version 8 specified in deployment.xml", e.getMessage()); } try { context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).build()); - fail("Should fail when compiled against a version which does not yet exist"); + fail("Should fail when compiled against a version which is only compatible with not-yet-existent versions"); } catch (IllegalArgumentException e) { - assertEquals("no suitable platform version found for package compiled against 8", e.getMessage()); + assertEquals("no platforms are compatible with compile version 8", e.getMessage()); } tester.controllerTester().upgradeSystem(version3); @@ -765,7 +765,7 @@ public class ControllerTest { fail("Should fail when specifying a major which is incompatible with compile version"); } catch (IllegalArgumentException e) { - assertEquals("Will not start dev-us-east-1 for tenant.application with incompatible platform version (8) and compile versions (7)", e.getMessage()); + assertEquals("no platforms on major version 8 specified in deployment.xml are compatible with compile version 7", e.getMessage()); } context.runJob(zone, new ApplicationPackageBuilder().compileVersion(version3).majorVersion(8).build()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index b3db4a8b845..347f1d4ab15 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -220,12 +220,11 @@ public class DeploymentContext { .allMatch(deployments -> deployments.stream() .allMatch(deployment -> deployment.version().equals(version)))); - for (var spec : application().deploymentSpec().instances()) - for (JobType type : new DeploymentSteps(spec, tester.controller().zoneRegistry()).productionJobs()) - assertTrue(tester.configServer().nodeRepository() - .list(type.zone(), - NodeFilter.all().applications(applicationId.defaultInstance())).stream() - .allMatch(node -> node.currentVersion().equals(version))); + for (JobId job : deploymentStatus().jobs().matching(job -> job.id().type().isProduction()).mapToList(JobStatus::id)) + assertTrue(tester.configServer().nodeRepository() + .list(job.type().zone(), + NodeFilter.all().applications(job.application())).stream() + .allMatch(node -> node.currentVersion().equals(version))); assertFalse(instance().change().hasTargets()); return this; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java index c9e86849ed8..38ebe030e8e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -254,7 +254,7 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { "some-id", "description", new BigDecimal("123.00"), - "some-plan", + "paid", "Smith", addedAt ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index 1235dfb33b7..58701732515 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -76,12 +76,13 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { @Test public void require_admin_for_update_plan() { var request = request("/billing/v2/tenant/" + tenant.value(), Request.Method.PATCH) - .data("{\"plan\": \"pay-as-you-go\"}"); + .data("{\"plan\": \"paid\"}"); var forbidden = request.roles(tenantReader); tester.assertResponse(forbidden, ACCESS_DENIED, 403); var success = request.roles(tenantAdmin); - tester.assertResponse(success, "{\"tenant\":\"tenant1\",\"plan\":\"pay-as-you-go\",\"collection\":\"AUTO\"}"); + tester.assertResponse(success, """ + {"tenant":"tenant1","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"collection":"AUTO"}"""); } @Test @@ -93,7 +94,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { tester.assertResponse(forbidden, "{\"error-code\":\"FORBIDDEN\",\"message\":\"Only accountant can change billing method\"}", 403); var success = request.roles(financeAdmin); - tester.assertResponse(success, "{\"tenant\":\"tenant1\",\"plan\":\"trial\",\"collection\":\"INVOICE\"}"); + tester.assertResponse(success, """ + {"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"collection":"INVOICE"}"""); } @Test @@ -108,7 +110,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { tester.assertResponse(listRequest, "{\"invoices\":[{\"id\":\"id-1\",\"from\":\"2020-05-23\",\"to\":\"2020-05-28\",\"total\":\"123.00\",\"status\":\"OPEN\"}]}"); var singleRequest = request("/billing/v2/tenant/" + tenant + "/bill/id-1").roles(tenantReader); - tester.assertResponse(singleRequest, "{\"id\":\"id-1\",\"from\":\"2020-05-23\",\"to\":\"2020-05-28\",\"total\":\"123.00\",\"status\":\"OPEN\",\"statusHistory\":[{\"at\":\"2020-05-23T00:00:00Z\",\"status\":\"OPEN\"}],\"items\":[{\"id\":\"some-id\",\"description\":\"description\",\"amount\":\"123.00\",\"plan\":\"some-plan\",\"planName\":\"Plan with id: some-plan\",\"cpu\":{},\"memory\":{},\"disk\":{}}]}"); + tester.assertResponse(singleRequest, """ + {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"cpu":{},"memory":{},"disk":{}}]}"""); } @Test @@ -120,7 +123,8 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { "}", 403); var accountantRequest = request("/billing/v2/accountant").roles(Role.hostedAccountant()); - tester.assertResponse(accountantRequest, "{\"tenants\":[{\"tenant\":\"tenant1\",\"plan\":\"trial\",\"collection\":\"AUTO\",\"lastBill\":null,\"unbilled\":\"0.00\"}]}"); + tester.assertResponse(accountantRequest, """ + {"tenants":[{"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"collection":"AUTO","lastBill":null,"unbilled":"0.00"}]}"""); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json index fe89ef246bb..e9b18a879b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json @@ -15,8 +15,8 @@ "id": "some-id", "description": "description", "amount": "123.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan" + "plan": "paid", + "planName": "Plan with id: paid" } ] }, @@ -46,8 +46,8 @@ "id": "some-id", "description": "description", "amount": "123.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan" + "plan": "paid", + "planName": "Plan with id: paid" } ] }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json index 953b946c329..adb319a3642 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json @@ -11,8 +11,8 @@ "id": "some-id", "description": "description", "amount": "123.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan" + "plan": "paid", + "planName": "Plan with id: paid" } ] }, @@ -37,8 +37,8 @@ "id": "some-id", "description": "description", "amount": "123.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan" + "plan": "paid", + "planName": "Plan with id: paid" } ] } diff --git a/dist/vespa.spec b/dist/vespa.spec index d278204be2e..72a2b58f930 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -78,6 +78,7 @@ BuildRequires: gcc-toolset-11-libatomic-devel %define _devtoolset_enable /opt/rh/gcc-toolset-11/enable %endif BuildRequires: maven +BuildRequires: maven-openjdk17 BuildRequires: pybind11-devel BuildRequires: python3-pytest BuildRequires: python36-devel @@ -145,6 +146,7 @@ BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %if 0%{?el9} BuildRequires: cmake >= 3.20.2 BuildRequires: maven +BuildRequires: maven-openjdk17 BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.11.0 @@ -162,6 +164,9 @@ BuildRequires: gmock-devel %if 0%{?fedora} BuildRequires: cmake >= 3.9.1 BuildRequires: maven +%if %{?fedora} >= 35 && ! 0%{?amzn2022} +BuildRequires: maven-openjdk17 +%endif BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.11.0 @@ -687,12 +692,6 @@ exit 0 %postun base if [ $1 -eq 0 ]; then # this is an uninstallation rm -f /etc/profile.d/vespa.sh -%if %{_create_vespa_user} - ! getent passwd %{_vespa_user} >/dev/null || userdel %{_vespa_user} -%endif -%if %{_create_vespa_group} - ! getent group %{_vespa_group} >/dev/null || groupdel %{_vespa_group} -%endif fi # Keep modifications to conf/vespa/default-env.txt across # package uninstall + install. @@ -715,11 +714,14 @@ fi %doc %dir %{_prefix} %{_prefix}/bin +%exclude %{_prefix}/bin/vespa %exclude %{_prefix}/bin/vespa-destination %exclude %{_prefix}/bin/vespa-document-statistics %exclude %{_prefix}/bin/vespa-fbench +%exclude %{_prefix}/bin/vespa-feed-client %exclude %{_prefix}/bin/vespa-feeder %exclude %{_prefix}/bin/vespa-get +%exclude %{_prefix}/bin/vespa-jvm-dumper %exclude %{_prefix}/bin/vespa-logfmt %exclude %{_prefix}/bin/vespa-query-profile-dump-tool %exclude %{_prefix}/bin/vespa-stat @@ -742,7 +744,6 @@ fi %{_prefix}/include %dir %{_prefix}/lib %dir %{_prefix}/lib/jars -%{_prefix}/lib/jars/application-model-jar-with-dependencies.jar %{_prefix}/lib/jars/application-preprocessor-jar-with-dependencies.jar %{_prefix}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar %{_prefix}/lib/jars/cloud-tenant-cd-jar-with-dependencies.jar @@ -774,6 +775,7 @@ fi %{_prefix}/libexec %exclude %{_prefix}/libexec/vespa_ann_benchmark %exclude %{_prefix}/libexec/vespa/common-env.sh +%exclude %{_prefix}/libexec/vespa/find-pid %exclude %{_prefix}/libexec/vespa/node-admin.sh %exclude %{_prefix}/libexec/vespa/standalone-container.sh %exclude %{_prefix}/libexec/vespa/vespa-curl-wrapper diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 3dca80885f7..960f15eac27 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -82,6 +82,7 @@ vespa_define_module( src/tests/instruction/sparse_full_overlap_join_function src/tests/instruction/sparse_merge_function src/tests/instruction/sparse_no_overlap_join_function + src/tests/instruction/sparse_singledim_lookup src/tests/instruction/sum_max_dot_product_function src/tests/instruction/unpack_bits_function src/tests/instruction/vector_from_doubles_function diff --git a/eval/src/tests/instruction/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp b/eval/src/tests/instruction/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp index 11654a38342..5cd63ce0492 100644 --- a/eval/src/tests/instruction/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp +++ b/eval/src/tests/instruction/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp @@ -56,10 +56,10 @@ TEST("require that tensor peek can be optimized for dense tensors") { TEST_DO(verify("x3y2f{x:(a-1),y:(b)}", 0.0, 1, 0)); } -TEST("require that tensor peek is not optimized for sparse tensor") { +TEST("require that tensor peek is optimized differently for sparse tensor") { TEST_DO(verify("xm{x:1}", 1.0, 0, 1)); - TEST_DO(verify("xm{x:(c)}", 3.0, 0, 1)); - TEST_DO(verify("xm{x:(c+1)}", 0.0, 0, 1)); + TEST_DO(verify("xm{x:(c)}", 3.0, 0, 0)); + TEST_DO(verify("xm{x:(c+1)}", 0.0, 0, 0)); } TEST("require that tensor peek is not optimized for mixed tensor") { @@ -71,10 +71,10 @@ TEST("require that tensor peek is not optimized for mixed tensor") { TEST("require that indexes are truncated when converted to integers") { TEST_DO(verify("x3{x:(a+0.7)}", 2.0, 1, 0)); TEST_DO(verify("x3{x:(a+0.3)}", 2.0, 1, 0)); - TEST_DO(verify("xm{x:(a+0.7)}", 1.0, 0, 1)); - TEST_DO(verify("xm{x:(a+0.3)}", 1.0, 0, 1)); - TEST_DO(verify("xm{x:(-a-0.7)}", 4.0, 0, 1)); - TEST_DO(verify("xm{x:(-a-0.3)}", 4.0, 0, 1)); + TEST_DO(verify("xm{x:(a+0.7)}", 1.0, 0, 0)); + TEST_DO(verify("xm{x:(a+0.3)}", 1.0, 0, 0)); + TEST_DO(verify("xm{x:(-a-0.7)}", 4.0, 0, 0)); + TEST_DO(verify("xm{x:(-a-0.3)}", 4.0, 0, 0)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/instruction/sparse_singledim_lookup/CMakeLists.txt b/eval/src/tests/instruction/sparse_singledim_lookup/CMakeLists.txt new file mode 100644 index 00000000000..983fd717540 --- /dev/null +++ b/eval/src/tests/instruction/sparse_singledim_lookup/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_sparse_singledim_lookup_test_app TEST + SOURCES + sparse_singledim_lookup_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_sparse_singledim_lookup_test_app COMMAND eval_sparse_singledim_lookup_test_app) diff --git a/eval/src/tests/instruction/sparse_singledim_lookup/sparse_singledim_lookup_test.cpp b/eval/src/tests/instruction/sparse_singledim_lookup/sparse_singledim_lookup_test.cpp new file mode 100644 index 00000000000..7d4e985a5ce --- /dev/null +++ b/eval/src/tests/instruction/sparse_singledim_lookup/sparse_singledim_lookup_test.cpp @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/instruction/sparse_singledim_lookup.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +//----------------------------------------------------------------------------- + +struct FunInfo { + using LookFor = SparseSingledimLookup; + void verify(const LookFor &fun) const { + EXPECT_TRUE(fun.result_is_mutable()); + } +}; + +void verify_optimized(const vespalib::string &expr) { + CellTypeSpace type_space(CellTypeUtils::list_types(), 1); + EvalFixture::verify<FunInfo>(expr, {FunInfo()}, type_space); +} + +void verify_not_optimized(const vespalib::string &expr) { + CellTypeSpace just_float({CellType::FLOAT}, 1); + EvalFixture::verify<FunInfo>(expr, {}, just_float); +} + +//----------------------------------------------------------------------------- + +TEST(SparseSingledimLookup, expression_can_be_optimized) { + verify_optimized("x5_1{x:(1+2)}"); +} + +TEST(SparseSingledimLookup, optimized_expression_handles_failed_lookup) { + verify_optimized("x5_1{x:(5+5)}"); + verify_optimized("x5_1{x:(5-10)}"); +} + +TEST(SparseSingledimLookup, verbatim_expression_is_not_optimized) { + verify_not_optimized("x5_1{x:3}"); + verify_not_optimized("x5_1{x:(3)}"); +} + +TEST(SparseSingledimLookup, similar_expressions_are_not_optimized) { + verify_not_optimized("x5{x:(1+2)}"); + verify_not_optimized("x5_1y3{x:(1+2)}"); + verify_not_optimized("x5_1y3_1{x:(1+2)}"); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index b6258acc9cb..45a4c79d2ed 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -9,6 +9,7 @@ #include <vespa/eval/instruction/sparse_112_dot_product.h> #include <vespa/eval/instruction/mixed_112_dot_product.h> #include <vespa/eval/instruction/sparse_merge_function.h> +#include <vespa/eval/instruction/sparse_singledim_lookup.h> #include <vespa/eval/instruction/sparse_no_overlap_join_function.h> #include <vespa/eval/instruction/sparse_full_overlap_join_function.h> #include <vespa/eval/instruction/mixed_inner_product_function.h> @@ -100,6 +101,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &, const Te child.set(SparseMergeFunction::optimize(child.get(), stash)); child.set(SparseNoOverlapJoinFunction::optimize(child.get(), stash)); child.set(SparseFullOverlapJoinFunction::optimize(child.get(), stash)); + child.set(SparseSingledimLookup::optimize(child.get(), stash)); }); return root.get(); } diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index c6700b0565a..c5cc99d9137 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -402,6 +402,7 @@ public: // mapping from dimension name to verbatim label or child index: using Spec = std::map<vespalib::string, LabelOrChildIndex>; Spec make_spec() const; + const TensorFunction ¶m() const { return _param.get(); } const ValueType ¶m_type() const { return _param.get().result_type(); } bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const final override; diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index f1ec8aa49a9..c4fd0443cbb 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -42,6 +42,7 @@ vespa_add_library(eval_instruction OBJECT sparse_full_overlap_join_function.cpp sparse_merge_function.cpp sparse_no_overlap_join_function.cpp + sparse_singledim_lookup.cpp sum_max_dot_product_function.cpp unpack_bits_function.cpp vector_from_doubles_function.cpp diff --git a/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp new file mode 100644 index 00000000000..4dc453d2a58 --- /dev/null +++ b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.cpp @@ -0,0 +1,84 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_singledim_lookup.h" +#include <vespa/eval/eval/fast_value.hpp> +#include <vespa/vespalib/util/typify.h> + +namespace vespalib::eval { + +using namespace tensor_function; +using namespace operation; +using namespace instruction; +using Handle = SharedStringRepo::Handle; + +namespace { + +template <typename CT> +double my_sparse_singledim_lookup_fallback(const Value::Index &idx, const CT *cells, string_id key) __attribute__((noinline)); +template <typename CT> +double my_sparse_singledim_lookup_fallback(const Value::Index &idx, const CT *cells, string_id key) { + size_t subspace = 0; + const string_id *key_ref = &key; + auto view = idx.create_view(ConstArrayRef<size_t>{&subspace, 1}); + view->lookup(ConstArrayRef<const string_id *>{&key_ref, 1}); + if (!view->next_result({}, subspace)) { + return 0.0; + } + return cells[subspace]; +} + +template <typename CT> +double my_fast_sparse_singledim_lookup(const FastAddrMap *map, const CT *cells, string_id key) +{ + auto subspace = map->lookup_singledim(key); + return (subspace != FastAddrMap::npos()) ? double(cells[subspace]) : 0.0; +} + +template <typename CT> +void my_sparse_singledim_lookup_op(InterpretedFunction::State &state, uint64_t) { + const auto &idx = state.peek(1).index(); + const CT *cells = state.peek(1).cells().typify<CT>().cbegin(); + int64_t number(state.peek(0).as_double()); + Handle handle = Handle::handle_from_number(number); + double result = __builtin_expect(is_fast(idx), true) + ? my_fast_sparse_singledim_lookup<CT>(&as_fast(idx).map, cells, handle.id()) + : my_sparse_singledim_lookup_fallback<CT>(idx, cells, handle.id()); + state.pop_pop_push(state.stash.create<DoubleValue>(result)); +} + +struct MyGetFun { + template <typename CT> + static auto invoke() { return my_sparse_singledim_lookup_op<CT>; } +}; + +} // namespace <unnamed> + +SparseSingledimLookup::SparseSingledimLookup(const TensorFunction &tensor, + const TensorFunction &expr) + : tensor_function::Op2(ValueType::double_type(), tensor, expr) +{ +} + +InterpretedFunction::Instruction +SparseSingledimLookup::compile_self(const ValueBuilderFactory &, Stash &) const +{ + auto op = typify_invoke<1,TypifyCellType,MyGetFun>(lhs().result_type().cell_type()); + return InterpretedFunction::Instruction(op); +} + +const TensorFunction & +SparseSingledimLookup::optimize(const TensorFunction &expr, Stash &stash) +{ + auto peek = as<Peek>(expr); + if (peek && peek->result_type().is_double() && + peek->param_type().is_sparse() && + (peek->param_type().dimensions().size() == 1) && + (peek->map().size() == 1) && + (std::holds_alternative<TensorFunction::Child>(peek->map().begin()->second))) + { + return stash.create<SparseSingledimLookup>(peek->param(), std::get<TensorFunction::Child>(peek->map().begin()->second).get()); + } + return expr; +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h new file mode 100644 index 00000000000..8c79860361a --- /dev/null +++ b/eval/src/vespa/eval/instruction/sparse_singledim_lookup.h @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::eval { + +/** + * Look up the result of an expression (double->int64_t->label_enum) + * in a sparse tensor with a single dimension, resulting in a double + * result. If lookup keys are kept small [0,10000000> (to avoid label + * enumeration) this is a simple hashtable lookup with numeric keys. + **/ +class SparseSingledimLookup : public tensor_function::Op2 +{ +public: + SparseSingledimLookup(const TensorFunction &tensor, const TensorFunction &expr); + InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + bool result_is_mutable() const override { return true; } + static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); +}; + +} // namespace diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 39b555ea105..8bc98cf0733 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -36,7 +36,8 @@ <target_jdk_version>17</target_jdk_version> <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version> - <junit.version>5.8.1</junit.version> <!-- NOTE: this must be in sync with junit version specified in 'tenant-cd-api' --> + <!-- NOTE: this must not be overriden by users, and must be in sync with junit version specified in 'tenant-cd-api' --> + <vespa.junit.version>5.8.1</vespa.junit.version> <test.categories>!integration</test.categories> <!-- To allow specialized base pom to include additional "test provided" dependencies --> @@ -94,13 +95,13 @@ <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> - <version>${junit.version}</version> + <version>${vespa.junit.version}</version> </dependency> <dependency> <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> - <version>${junit.version}</version> + <version>${vespa.junit.version}</version> </dependency> </dependencies> </dependencyManagement> @@ -235,8 +236,8 @@ <exclude>org.junit.jupiter:junit-jupiter-api:*:jar:compile</exclude> <exclude>org.junit.jupiter:junit-jupiter-api:*:jar:provided</exclude> <exclude>org.junit.jupiter:junit-jupiter-api:*:jar:runtime</exclude> - <exclude>org.junit.jupiter:junit-jupiter-api:(,${junit.version}):jar:*</exclude> - <exclude>org.junit.jupiter:junit-jupiter-api:(${junit.version},):jar:*</exclude> + <exclude>org.junit.jupiter:junit-jupiter-api:(,${vespa.junit.version}):jar:*</exclude> + <exclude>org.junit.jupiter:junit-jupiter-api:(${vespa.junit.version},):jar:*</exclude> </excludes> </bannedDependencies> </rules> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index d6ef152dcb2..b3ff0c42547 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -89,7 +89,7 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { } } - /** Returns the last time application was deployed. Epoch is returned if the application has never been deployed. */ + /** Returns the last time application was activated. Epoch is returned if the application has never been deployed. */ protected final Instant getLastDeployTime(ApplicationId application) { return deployer.lastDeployTime(application).orElse(Instant.EPOCH); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index a461d98697d..9c6eb2199f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -204,7 +204,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toMap(Node::hostname, Function.identity()))); nodes.stream() - .filter(node -> node.allocation().isPresent() && !node.status().wantToDeprovision()) + .filter(node -> node.allocation().isPresent()) .flatMap(node -> node.parentHostname().stream()) .distinct() .forEach(hostsByHostname::remove); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 5e703139f41..f32fd225427 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -125,7 +125,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { DefaultTimes(Zone zone, Deployer deployer) { autoscalingInterval = Duration.ofMinutes(15); - dynamicProvisionerInterval = Duration.ofMinutes(5); + dynamicProvisionerInterval = Duration.ofMinutes(3); failedExpirerInterval = Duration.ofMinutes(10); failGrace = Duration.ofMinutes(30); infrastructureProvisionInterval = Duration.ofMinutes(3); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 451bcfb0b78..30d0f673fe1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -574,25 +574,6 @@ public class DynamicProvisioningMaintainerTest { assertEquals(2, provisioningTester.activate(applicationId, prepared).size()); } - @Test - public void deprovision_parked_node_with_allocation() { - var tester = new DynamicProvisioningTester(); - tester.hostProvisioner.with(Behaviour.failProvisioning); - Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.parked); - Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.parked, DynamicProvisioningTester.tenantApp); - tester.nodeRepository.nodes().deprovision("host4", Agent.operator, Instant.now()); - - assertEquals(Optional.of(true), tester.nodeRepository.nodes().node("host4").map(n -> n.status().wantToDeprovision())); - assertEquals(Optional.of(Node.State.parked), tester.nodeRepository.nodes().node("host4").map(Node::state)); - assertEquals(Optional.of(true), tester.nodeRepository.nodes().node("host4-1").map(n -> n.status().wantToDeprovision())); - assertEquals(Optional.of(Node.State.parked), tester.nodeRepository.nodes().node("host4-1").map(Node::state)); - - tester.maintainer.maintain(); - - assertEquals(Optional.empty(), tester.nodeRepository.nodes().node("host4")); - assertEquals(Optional.empty(), tester.nodeRepository.nodes().node("host4-1")); - } - private void assertCfghost3IsActive(DynamicProvisioningTester tester) { assertEquals(5, tester.nodeRepository.nodes().list(Node.State.active).size()); assertEquals(3, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.confighost).size()); diff --git a/storage/src/vespa/storage/distributor/update_metric_set.cpp b/storage/src/vespa/storage/distributor/update_metric_set.cpp index 70374cab6d2..82f55d3e819 100644 --- a/storage/src/vespa/storage/distributor/update_metric_set.cpp +++ b/storage/src/vespa/storage/distributor/update_metric_set.cpp @@ -7,7 +7,7 @@ namespace storage { using metrics::MetricSet; UpdateMetricSet::UpdateMetricSet(MetricSet* owner) - : PersistenceOperationMetricSet("updates.sum", owner), + : PersistenceOperationMetricSet("updates", owner), diverging_timestamp_updates("diverging_timestamp_updates", {}, "Number of updates that report they were performed against " "divergent version timestamps on different replicas", this), diff --git a/storage/src/vespa/storage/distributor/visitormetricsset.cpp b/storage/src/vespa/storage/distributor/visitormetricsset.cpp index c449b82e93c..c94dc025fa1 100644 --- a/storage/src/vespa/storage/distributor/visitormetricsset.cpp +++ b/storage/src/vespa/storage/distributor/visitormetricsset.cpp @@ -7,7 +7,7 @@ namespace storage { using metrics::MetricSet; VisitorMetricSet::VisitorMetricSet(MetricSet* owner) - : PersistenceOperationMetricSet("visitor.sum", owner), + : PersistenceOperationMetricSet("visitor", owner), buckets_per_visitor("buckets_per_visitor", {}, "The number of sub buckets visited as part of a " "single client visitor command", this), |
