diff options
| author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-02-05 09:27:01 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-02-05 09:27:01 +0100 |
| commit | cc6c05c46211b0c061f99408232bd5740a625107 (patch) | |
| tree | ba148745574afa5833ec9d55e448d4bf9005b58e /config-model | |
| parent | 4285a9e76ddab7761031899d52259c1fc7779c4b (diff) | |
| parent | 2a7f13ab9f019586fc38275735ead7ad02afbd54 (diff) | |
Merge branch 'master' into balder/less-unused-header-body-references
Diffstat (limited to 'config-model')
38 files changed, 1598 insertions, 62 deletions
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index fc42864f1d0..41a30c4553d 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final StructDataType body; private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); private final Set<Name> documentReferences; + // Imported fields are virtual and therefore exist outside of the SD's document field definition + // block itself. But for features like imported fields in a non-search context (e.g. GC selections) + // it is necessary to know that certain identifiers refer to imported fields instead of being unknown + // document fields. To achieve this, we track the names of imported fields as part of the document + // config itself. + private final Set<String> importedFieldNames; public NewDocumentType(Name name) { this(name, emptySet()); } - public NewDocumentType(Name name, Set<Name> documentReferences) { + public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) { this( name, new StructDataType(name.getName() + ".header"), new StructDataType(name.getName() + ".body"), new FieldSets(), - documentReferences); + documentReferences, + importedFieldNames); + } + + public NewDocumentType(Name name, Set<Name> documentReferences) { + this(name, documentReferences, emptySet()); } public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs, - Set<Name> documentReferences) { + Set<Name> documentReferences, + Set<String> importedFieldNames) { super(name.getName()); this.name = name; this.header = header; @@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } } this.documentReferences = documentReferences; + this.importedFieldNames = importedFieldNames; } public Name getFullName() { @@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return documentReferences; } + public Set<String> getImportedFieldNames() { + return importedFieldNames; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index f0b1b427531..d3a78321106 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -338,7 +340,8 @@ public class DocumentModelBuilder { sdoc.getDocumentType().contentStruct(), sdoc.getDocumentType().getBodyType(), sdoc.getFieldSets(), - convertDocumentReferencesToNames(sdoc.getDocumentReferences())); + convertDocumentReferencesToNames(sdoc.getDocumentReferences()), + convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); @@ -404,6 +407,13 @@ public class DocumentModelBuilder { .collect(toSet()); } + private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { + if (importedFields == null) { + return emptySet(); + } + return Collections.unmodifiableSet(importedFields.fields().keySet()); + } + private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { for (Field f : fields) { DataType type = f.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java new file mode 100644 index 00000000000..ee4fc41f2f9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java @@ -0,0 +1,33 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; + +import java.util.Collections; +import java.util.List; + +/** + * Enumerates and emplaces a set of all imported fields into a SDDocumentType from + * its corresponding Search instance. + */ +public class ImportedFieldsEnumerator { + + private final List<Search> searchDefinitions; + + public ImportedFieldsEnumerator(List<Search> searchDefinitions) { + this.searchDefinitions = searchDefinitions; + } + + public void enumerateImportedFields(SDDocumentType documentType) { + var search = this.searchDefinitions.stream() + .filter(s -> s.getDocument() != null) + .filter(s -> s.getDocument().getName().equals(documentType.getName())) + .findFirst(); + if (search.isEmpty()) { + return; // No imported fields present. + } + search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index fa196cd3bbf..23eb814de81 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -788,8 +788,7 @@ public class RankProfile implements Cloneable { type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() + " with type " + field.getType().asTensorType() + - ", but this is already defined " + - "in another query profile with type " + + ", but this is already defined in another query profile with type " + context.getType(feature.get()))); context.setType(feature.get(), type); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 949539ff99f..eb68e6af203 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -230,20 +230,22 @@ public class SearchBuilder { sdocs.add(search.getDocument()); } } - SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); + var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); orderer.process(); for (SDDocumentType sdoc : orderer.getOrdered()) { new FieldOperationApplierForStructs().process(sdoc); new FieldOperationApplier().process(sdoc); } - DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList); + var resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); if (validate) new DocumentGraphValidator().validateDocumentGraph(sdocs); - DocumentModelBuilder builder = new DocumentModelBuilder(model); + var builder = new DocumentModelBuilder(model); for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list? process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index 414a605c621..b3f98cc6f26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable { private FieldSets fieldSets; // Document references private Optional<DocumentReferences> documentReferences = Optional.empty(); + private TemporaryImportedFields temporaryImportedFields; static { VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); @@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable { type.docType = docType.clone(); type.inheritedTypes.putAll(inheritedTypes); type.structType = structType; + // TODO this isn't complete; should it be..?! return type; } @@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable { this.documentReferences = Optional.of(documentReferences); } + public TemporaryImportedFields getTemporaryImportedFields() { + return temporaryImportedFields; + } + + public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { + this.temporaryImportedFields = temporaryImportedFields; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index 6fdf448a39b..a6707ec7ac0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -27,6 +27,7 @@ public class ExpressionTransforms { ImmutableList.of(new TensorFlowFeatureConverter(), new OnnxFeatureConverter(), new XgboostFeatureConverter(), + new LightGBMFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), new FunctionInliner(), diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java new file mode 100644 index 00000000000..5bde627dc0a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java @@ -0,0 +1,59 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; + +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Replaces instances of the lightgbm(model-path) pseudofeature with the + * native Vespa ranking expression implementing the same computation. + * + * @author lesters + */ +public class LightGBMFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> { + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map<Path, ConvertedModel> convertedLightGBMModels = new HashMap<>(); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("lightgbm")) return feature; + + try { + FeatureArguments arguments = asFeatureArguments(feature.getArguments()); + ConvertedModel convertedModel = + convertedLightGBMModels.computeIfAbsent(arguments.path(), + path -> ConvertedModel.fromSourceOrStore(path, true, context)); + return convertedModel.expression(arguments, context); + } catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e); + } + } + + private FeatureArguments asFeatureArguments(Arguments arguments) { + if (arguments.size() != 1) + throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " + + "the LightGBM model file under [application]/models"); + return new FeatureArguments(arguments); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index d6c334ee80b..6d3de23238d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor { } private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { + ImmutableSDField targetField, boolean validate) { TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); @@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor { } private void validateTargetField(TemporaryImportedField importedField, - ImmutableSDField targetField, DocumentReference reference) { + ImmutableSDField targetField, DocumentReference reference) { if (!targetField.doesAttributing()) { fail(importedField, targetFieldAsString(targetField.getName(), reference) + ": Is not an attribute field. Only attribute fields supported"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java index 9a9e9bbba5f..89b8889b4ae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java @@ -15,7 +15,10 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.logging.Level; /** @@ -45,9 +48,10 @@ public class RankingExpressionTypeResolver extends Processor { public void process(boolean validate, boolean documentsOnly) { if (documentsOnly) return; + Set<Reference> warnedAbout = new HashSet<>(); for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { - resolveTypesIn(profile, validate); + resolveTypesIn(profile, validate, warnedAbout); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("In " + (search != null ? search + ", " : "") + profile, e); @@ -60,7 +64,7 @@ public class RankingExpressionTypeResolver extends Processor { * * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types */ - private void resolveTypesIn(RankProfile profile, boolean validate) { + private void resolveTypesIn(RankProfile profile, boolean validate, Set<Reference> warnedAbout) { MapEvaluationTypeContext context = profile.typeContext(queryProfiles); for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) { ExpressionFunction expressionFunction = function.getValue().function(); @@ -83,10 +87,14 @@ public class RankingExpressionTypeResolver extends Processor { profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); - if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty()) { - deployLogger.log(Level.WARNING, "The following query features are not declared in query profile " + + if ( context.tensorsAreUsed() && + ! context.queryFeaturesNotDeclared().isEmpty() && + ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { + deployLogger.log(Level.WARNING, "The following query features used in '" + profile.getName() + + "' are not declared in query profile " + "types and will be interpreted as scalars, not tensors: " + context.queryFeaturesNotDeclared()); + warnedAbout.addAll(context.queryFeaturesNotDeclared()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 02d500931d7..e0d015cc8b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -62,6 +62,7 @@ public class DocumentManager { } } } + private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) { for (AnnotationType type : types) { DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder(); @@ -110,6 +111,7 @@ public class DocumentManager { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } buildConfig(dt.getFieldSets(), doc); + buildImportedFieldsConfig(dt.getImportedFieldNames(), doc); } else if (type instanceof TemporaryStructuredDataType) { //Ignored } else if (type instanceof StructDataType) { @@ -164,4 +166,12 @@ public class DocumentManager { doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); } + private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index d4228b52746..e6bf826dccc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -58,6 +58,7 @@ public class DocumentTypes { buildConfig(annotation, atb); } buildConfig(documentType.getFieldSets(), db); + buildImportedFieldsConfig(documentType.getImportedFieldNames(), db); builder.documenttype(db); } @@ -120,6 +121,14 @@ public class DocumentTypes { } } + private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + private void buildConfig(StructDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 57d938f8a71..62662bde2ad 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -72,6 +72,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.server.network.tls-connections-broken")); metrics.add(new Metric("vds.server.network.failed-tls-config-reloads")); + // C++ Fnet metrics + metrics.add(new Metric("vds.server.fnet.num-connections")); + return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java index 4f84a01ff94..4a331718985 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java @@ -8,6 +8,7 @@ import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.util.List; import java.util.Optional; import static com.yahoo.component.ComponentSpecification.fromString; @@ -16,6 +17,7 @@ import static com.yahoo.component.ComponentSpecification.fromString; * Configure SSL using file references * * @author mortent + * @author bjorncs */ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements ConnectorConfig.Producer { public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@"; @@ -26,8 +28,16 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C private final String certificatePath; private final String caCertificatePath; private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; + private final List<String> cipherSuites; + private final List<String> protocolVersions; - public ConfiguredFilebasedSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) { + public ConfiguredFilebasedSslProvider(String servername, + String privateKeyPath, + String certificatePath, + String caCertificatePath, + String clientAuthentication, + List<String> cipherSuites, + List<String> protocolVersions) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), fromString(COMPONENT_CLASS), @@ -36,15 +46,21 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C this.certificatePath = certificatePath; this.caCertificatePath = caCertificatePath; this.clientAuthentication = mapToConfigEnum(clientAuthentication); + this.cipherSuites = cipherSuites; + this.protocolVersions = protocolVersions; } @Override public void getConfig(ConnectorConfig.Builder builder) { - builder.ssl.enabled(true); - builder.ssl.privateKeyFile(privateKeyPath); - builder.ssl.certificateFile(certificatePath); - builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); - builder.ssl.clientAuth(clientAuthentication); + builder.ssl( + new ConnectorConfig.Ssl.Builder() + .enabled(true) + .privateKeyFile(privateKeyPath) + .certificateFile(certificatePath) + .caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")) + .clientAuth(clientAuthentication) + .enabledCipherSuites(cipherSuites) + .enabledProtocols(protocolVersions)); } public SimpleComponent getComponent() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index db831a1ec2f..562026ab4dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -9,13 +9,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; -import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.ConfiguredFilebasedSslProvider; +import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; import org.w3c.dom.Element; +import java.util.Arrays; +import java.util.List; import java.util.Optional; +import static java.util.stream.Collectors.toList; + /** * @author Einar M R Rosenvinge * @author mortent @@ -40,12 +44,16 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file")); Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file"); Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication"); + List<String> cipherSuites = extractOptionalCommaSeparatedList(sslConfigurator, "cipher-suites"); + List<String> protocols = extractOptionalCommaSeparatedList(sslConfigurator, "protocols"); return new ConfiguredFilebasedSslProvider( serverName, privateKeyFile, certificateFile, caCertificateFile.orElse(null), - clientAuthentication.orElse(null)); + clientAuthentication.orElse(null), + cipherSuites, + protocols); } else if (sslProviderConfigurator != null) { String className = sslProviderConfigurator.getAttribute("class"); String bundle = sslProviderConfigurator.getAttribute("bundle"); @@ -55,4 +63,13 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil } } + private static List<String> extractOptionalCommaSeparatedList(Element sslElement, String listElementName) { + return XmlHelper.getOptionalChildValue(sslElement, listElementName) + .map(element -> + Arrays.stream(element.split(",")) + .filter(listEntry -> !listEntry.isBlank()) + .map(String::trim) + .collect(toList())) + .orElse(List.of()); + } } diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index e3ad942e7b3..c47983adc12 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -37,7 +37,8 @@ OptionalDedicatedNodes = element nodes { attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & attribute dedicated { xsd:boolean }? & - attribute exclusive { xsd:boolean }? + attribute exclusive { xsd:boolean }? & + Resources? } GenericConfig = element config { diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 6e4346d96ee..726fa849c00 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -96,7 +96,9 @@ Ssl = element ssl { element private-key-file { string } & element certificate-file { string } & element ca-certificates-file { string }? & - element client-authentication { string "disabled" | string "want" | string "need" }? + element client-authentication { string "disabled" | string "want" | string "need" }? & + element cipher-suites { string }? & + element protocols { string }? } SslProvider = element ssl-provider { diff --git a/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd index ab5e42f983d..247df8a0241 100644 --- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd @@ -33,8 +33,12 @@ search test { expression: xgboost("xgboost_2_2") } + function my_lightgbm() { + expression: lightgbm("lightgbm_regression") + } + first-phase { - expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + my_lightgbm } } diff --git a/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7b50176625d --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -0,0 +1,104 @@ +enablecompression false +datatype[0].id 1381038251 +datatype[0].structtype[0].name "position" +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].compresstype NONE +datatype[0].structtype[0].compresslevel 0 +datatype[0].structtype[0].compressthreshold 95 +datatype[0].structtype[0].compressminsize 800 +datatype[0].structtype[0].field[0].name "x" +datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" +datatype[0].structtype[0].field[1].name "y" +datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" +datatype[1].id 595216861 +datatype[1].referencetype[0].target_type_id -1318255918 +datatype[2].id 542332920 +datatype[2].referencetype[0].target_type_id 443162583 +datatype[3].id 959075962 +datatype[3].structtype[0].name "ad.header" +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].compresstype NONE +datatype[3].structtype[0].compresslevel 0 +datatype[3].structtype[0].compressthreshold 95 +datatype[3].structtype[0].compressminsize 800 +datatype[3].structtype[0].field[0].name "campaign_ref" +datatype[3].structtype[0].field[0].datatype 595216861 +datatype[3].structtype[0].field[0].detailedtype "" +datatype[3].structtype[0].field[1].name "person_ref" +datatype[3].structtype[0].field[1].datatype 542332920 +datatype[3].structtype[0].field[1].detailedtype "" +datatype[4].id -255288561 +datatype[4].structtype[0].name "ad.body" +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].compresstype NONE +datatype[4].structtype[0].compresslevel 0 +datatype[4].structtype[0].compressthreshold 95 +datatype[4].structtype[0].compressminsize 800 +datatype[5].id 2987301 +datatype[5].documenttype[0].name "ad" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct 959075962 +datatype[5].documenttype[0].bodystruct -255288561 +datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[5].documenttype[0].importedfield[0].name "my_cool_field" +datatype[5].documenttype[0].importedfield[1].name "my_swag_field" +datatype[5].documenttype[0].importedfield[2].name "my_name" +datatype[6].id -2041471955 +datatype[6].structtype[0].name "campaign.header" +datatype[6].structtype[0].version 0 +datatype[6].structtype[0].compresstype NONE +datatype[6].structtype[0].compresslevel 0 +datatype[6].structtype[0].compressthreshold 95 +datatype[6].structtype[0].compressminsize 800 +datatype[6].structtype[0].field[0].name "cool_field" +datatype[6].structtype[0].field[0].datatype 2 +datatype[6].structtype[0].field[0].detailedtype "" +datatype[6].structtype[0].field[1].name "swag_field" +datatype[6].structtype[0].field[1].datatype 4 +datatype[6].structtype[0].field[1].detailedtype "" +datatype[7].id 1448849794 +datatype[7].structtype[0].name "campaign.body" +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].compresstype NONE +datatype[7].structtype[0].compresslevel 0 +datatype[7].structtype[0].compressthreshold 95 +datatype[7].structtype[0].compressminsize 800 +datatype[8].id -1318255918 +datatype[8].documenttype[0].name "campaign" +datatype[8].documenttype[0].version 0 +datatype[8].documenttype[0].inherits[0].name "document" +datatype[8].documenttype[0].inherits[0].version 0 +datatype[8].documenttype[0].headerstruct -2041471955 +datatype[8].documenttype[0].bodystruct 1448849794 +datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" +datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" +datatype[9].id 3129224 +datatype[9].structtype[0].name "person.header" +datatype[9].structtype[0].version 0 +datatype[9].structtype[0].compresstype NONE +datatype[9].structtype[0].compresslevel 0 +datatype[9].structtype[0].compressthreshold 95 +datatype[9].structtype[0].compressminsize 800 +datatype[9].structtype[0].field[0].name "name" +datatype[9].structtype[0].field[0].datatype 2 +datatype[9].structtype[0].field[0].detailedtype "" +datatype[10].id -2003767395 +datatype[10].structtype[0].name "person.body" +datatype[10].structtype[0].version 0 +datatype[10].structtype[0].compresstype NONE +datatype[10].structtype[0].compresslevel 0 +datatype[10].structtype[0].compressthreshold 95 +datatype[10].structtype[0].compressminsize 800 +datatype[11].id 443162583 +datatype[11].documenttype[0].name "person" +datatype[11].documenttype[0].version 0 +datatype[11].documenttype[0].inherits[0].name "document" +datatype[11].documenttype[0].inherits[0].version 0 +datatype[11].documenttype[0].headerstruct 3129224 +datatype[11].documenttype[0].bodystruct -2003767395 +datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7859703ffe0 --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -0,0 +1,141 @@ +enablecompression false +documenttype[0].id 2987301 +documenttype[0].name "ad" +documenttype[0].version 0 +documenttype[0].headerstruct 959075962 +documenttype[0].bodystruct -255288561 +documenttype[0].inherits[0].id 8 +documenttype[0].datatype[0].id 959075962 +documenttype[0].datatype[0].type STRUCT +documenttype[0].datatype[0].array.element.id 0 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 +documenttype[0].datatype[0].wset.key.id 0 +documenttype[0].datatype[0].wset.createifnonexistent false +documenttype[0].datatype[0].wset.removeifzero false +documenttype[0].datatype[0].annotationref.annotation.id 0 +documenttype[0].datatype[0].sstruct.name "ad.header" +documenttype[0].datatype[0].sstruct.version 0 +documenttype[0].datatype[0].sstruct.compression.type NONE +documenttype[0].datatype[0].sstruct.compression.level 0 +documenttype[0].datatype[0].sstruct.compression.threshold 95 +documenttype[0].datatype[0].sstruct.compression.minsize 200 +documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref" +documenttype[0].datatype[0].sstruct.field[0].id 23963250 +documenttype[0].datatype[0].sstruct.field[0].datatype 595216861 +documenttype[0].datatype[0].sstruct.field[0].detailedtype "" +documenttype[0].datatype[0].sstruct.field[1].name "person_ref" +documenttype[0].datatype[0].sstruct.field[1].id 100779805 +documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 +documenttype[0].datatype[0].sstruct.field[1].detailedtype "" +documenttype[0].datatype[1].id -255288561 +documenttype[0].datatype[1].type STRUCT +documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].map.key.id 0 +documenttype[0].datatype[1].map.value.id 0 +documenttype[0].datatype[1].wset.key.id 0 +documenttype[0].datatype[1].wset.createifnonexistent false +documenttype[0].datatype[1].wset.removeifzero false +documenttype[0].datatype[1].annotationref.annotation.id 0 +documenttype[0].datatype[1].sstruct.name "ad.body" +documenttype[0].datatype[1].sstruct.version 0 +documenttype[0].datatype[1].sstruct.compression.type NONE +documenttype[0].datatype[1].sstruct.compression.level 0 +documenttype[0].datatype[1].sstruct.compression.threshold 95 +documenttype[0].datatype[1].sstruct.compression.minsize 200 +documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +documenttype[0].referencetype[0].id 595216861 +documenttype[0].referencetype[0].target_type_id -1318255918 +documenttype[0].referencetype[1].id 542332920 +documenttype[0].referencetype[1].target_type_id 443162583 +documenttype[0].importedfield[0].name "my_cool_field" +documenttype[0].importedfield[1].name "my_swag_field" +documenttype[0].importedfield[2].name "my_name" +documenttype[1].id -1318255918 +documenttype[1].name "campaign" +documenttype[1].version 0 +documenttype[1].headerstruct -2041471955 +documenttype[1].bodystruct 1448849794 +documenttype[1].inherits[0].id 8 +documenttype[1].datatype[0].id -2041471955 +documenttype[1].datatype[0].type STRUCT +documenttype[1].datatype[0].array.element.id 0 +documenttype[1].datatype[0].map.key.id 0 +documenttype[1].datatype[0].map.value.id 0 +documenttype[1].datatype[0].wset.key.id 0 +documenttype[1].datatype[0].wset.createifnonexistent false +documenttype[1].datatype[0].wset.removeifzero false +documenttype[1].datatype[0].annotationref.annotation.id 0 +documenttype[1].datatype[0].sstruct.name "campaign.header" +documenttype[1].datatype[0].sstruct.version 0 +documenttype[1].datatype[0].sstruct.compression.type NONE +documenttype[1].datatype[0].sstruct.compression.level 0 +documenttype[1].datatype[0].sstruct.compression.threshold 95 +documenttype[1].datatype[0].sstruct.compression.minsize 200 +documenttype[1].datatype[0].sstruct.field[0].name "cool_field" +documenttype[1].datatype[0].sstruct.field[0].id 1588702436 +documenttype[1].datatype[0].sstruct.field[0].datatype 2 +documenttype[1].datatype[0].sstruct.field[0].detailedtype "" +documenttype[1].datatype[0].sstruct.field[1].name "swag_field" +documenttype[1].datatype[0].sstruct.field[1].id 1691224741 +documenttype[1].datatype[0].sstruct.field[1].datatype 4 +documenttype[1].datatype[0].sstruct.field[1].detailedtype "" +documenttype[1].datatype[1].id 1448849794 +documenttype[1].datatype[1].type STRUCT +documenttype[1].datatype[1].array.element.id 0 +documenttype[1].datatype[1].map.key.id 0 +documenttype[1].datatype[1].map.value.id 0 +documenttype[1].datatype[1].wset.key.id 0 +documenttype[1].datatype[1].wset.createifnonexistent false +documenttype[1].datatype[1].wset.removeifzero false +documenttype[1].datatype[1].annotationref.annotation.id 0 +documenttype[1].datatype[1].sstruct.name "campaign.body" +documenttype[1].datatype[1].sstruct.version 0 +documenttype[1].datatype[1].sstruct.compression.type NONE +documenttype[1].datatype[1].sstruct.compression.level 0 +documenttype[1].datatype[1].sstruct.compression.threshold 95 +documenttype[1].datatype[1].sstruct.compression.minsize 200 +documenttype[1].fieldsets{[document]}.fields[0] "cool_field" +documenttype[1].fieldsets{[document]}.fields[1] "swag_field" +documenttype[2].id 443162583 +documenttype[2].name "person" +documenttype[2].version 0 +documenttype[2].headerstruct 3129224 +documenttype[2].bodystruct -2003767395 +documenttype[2].inherits[0].id 8 +documenttype[2].datatype[0].id 3129224 +documenttype[2].datatype[0].type STRUCT +documenttype[2].datatype[0].array.element.id 0 +documenttype[2].datatype[0].map.key.id 0 +documenttype[2].datatype[0].map.value.id 0 +documenttype[2].datatype[0].wset.key.id 0 +documenttype[2].datatype[0].wset.createifnonexistent false +documenttype[2].datatype[0].wset.removeifzero false +documenttype[2].datatype[0].annotationref.annotation.id 0 +documenttype[2].datatype[0].sstruct.name "person.header" +documenttype[2].datatype[0].sstruct.version 0 +documenttype[2].datatype[0].sstruct.compression.type NONE +documenttype[2].datatype[0].sstruct.compression.level 0 +documenttype[2].datatype[0].sstruct.compression.threshold 95 +documenttype[2].datatype[0].sstruct.compression.minsize 200 +documenttype[2].datatype[0].sstruct.field[0].name "name" +documenttype[2].datatype[0].sstruct.field[0].id 1160796772 +documenttype[2].datatype[0].sstruct.field[0].datatype 2 +documenttype[2].datatype[0].sstruct.field[0].detailedtype "" +documenttype[2].datatype[1].id -2003767395 +documenttype[2].datatype[1].type STRUCT +documenttype[2].datatype[1].array.element.id 0 +documenttype[2].datatype[1].map.key.id 0 +documenttype[2].datatype[1].map.value.id 0 +documenttype[2].datatype[1].wset.key.id 0 +documenttype[2].datatype[1].wset.createifnonexistent false +documenttype[2].datatype[1].wset.removeifzero false +documenttype[2].datatype[1].annotationref.annotation.id 0 +documenttype[2].datatype[1].sstruct.name "person.body" +documenttype[2].datatype[1].sstruct.version 0 +documenttype[2].datatype[1].sstruct.compression.type NONE +documenttype[2].datatype[1].sstruct.compression.level 0 +documenttype[2].datatype[1].sstruct.compression.threshold 95 +documenttype[2].datatype[1].sstruct.compression.minsize 200 +documenttype[2].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file diff --git a/config-model/src/test/integration/lightgbm/models/regression.json b/config-model/src/test/integration/lightgbm/models/regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/integration/lightgbm/models/regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java new file mode 100644 index 00000000000..2c13427760f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java @@ -0,0 +1,15 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * Convenience class for tests that need to set attribute properties on fields. + */ +public class AttributeUtils { + + public static void addAttributeAspect(SDField field) { + field.parseIndexingScript("{ attribute }"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index 8378ec811a5..e46208c770d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to bar and add another field SDField fooRefToBarField = new SDField ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); @@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to non-existing document bar SDField fooRefToBarField = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); fooDocument.addField(fooRefToBarField); @@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest { resolver.resolveReferences(fooDocument); } - private static void addAttributeAspect(SDField fooRefToBarField) { - fooRefToBarField.parseIndexingScript("{ attribute }"); - } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java new file mode 100644 index 00000000000..fcbb89b5c42 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -0,0 +1,66 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.DataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedField; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImportedFieldsEnumeratorTest { + + @Test + public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { + Search parentSearch = new Search(); + SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch); + var parentField = new SDField("their_field", DataType.INT); + AttributeUtils.addAttributeAspect(parentField); + parentDocument.addField(parentField); + parentSearch.addDocument(parentDocument); + + Search fooSearch = new Search(); + SDField fooRefToParent = new SDField( + "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToParent); + var fooImports = fooSearch.temporaryImportedFields().get(); + fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); + fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); + fooSearch.addDocument(fooDocument); + + Search barSearch = new Search(); + SDField barRefToParent = new SDField( + "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(barRefToParent); + var barImports = barSearch.temporaryImportedFields().get(); + barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); + SDDocumentType barDocument = new SDDocumentType("bar", barSearch); + barSearch.addDocument(barDocument); + + var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch)); + + enumerator.enumerateImportedFields(parentDocument); + assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent + + enumerator.enumerateImportedFields(fooDocument); + assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import")); + + enumerator.enumerateImportedFields(barDocument); + assertImportedFieldsAre(barDocument, List.of("my_cool_import")); + } + + private void assertImportedFieldsAre(SDDocumentType documentType, List<String> expectedNames) { + assertNotNull(documentType.getTemporaryImportedFields()); + var actualNames = documentType.getTemporaryImportedFields().fields().keySet(); + var expectedNameSet = new HashSet<>(expectedNames); + assertEquals(expectedNameSet, actualNames); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 08dd5148b29..0cd6674751e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -15,6 +15,7 @@ import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import java.util.HashMap; @@ -33,6 +34,7 @@ class RankProfileSearchFixture { private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter()); private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); private final QueryProfileRegistry queryProfileRegistry; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index 9e9ef2589be..a306e0f2c90 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -275,7 +275,7 @@ public class RankingExpressionTypeResolverTestCase { builder.build(true, logger); String message = logger.findMessage("The following query features"); assertNotNull(message); - assertEquals("WARNING: The following query features are not declared in query profile types and " + + assertEquals("WARNING: The following query features used in 'my_rank_profile' are not declared in query profile types and " + "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", message); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java new file mode 100644 index 00000000000..79d19371f1c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author lesters + */ +public class RankingExpressionWithLightGBMTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/"); + + private final static String lightGBMExpression = + "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + + @Test + public void testNestedLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))"); + search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile"); + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')"); + searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { + return fixtureWith(firstPhaseExpression, null, null, + new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, + String constant, + String field, + RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} + diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java new file mode 100644 index 00000000000..d0ee0523489 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; + +import java.io.IOException; + +/** + * Utility functions for testing generated configs for reference/imported fields. + */ +public abstract class AbstractReferenceFieldTestCase extends SearchDefinitionTestCase { + + private static String TEST_FOLDER = "src/test/configmodel/types/references/"; + + protected void assertDocumentConfigs(DocumentModel model, + String cfgFileSpec) throws IOException { + assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); + assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); + } + + protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { + DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); + assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); + } + + protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { + DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); + assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java new file mode 100644 index 00000000000..599ae77a456 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; + +public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase { + + @Test + public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException { + assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines( + "search ad {", + " document ad {", + " field campaign_ref type reference<campaign> { indexing: attribute }", + " field person_ref type reference<person> { indexing: attribute }", + " }", + " import field campaign_ref.cool_field as my_cool_field {}", + " import field campaign_ref.swag_field as my_swag_field {}", + " import field person_ref.name as my_name {}", + "}")), + "multiple_imported_fields"); + } + + private static class TestDocumentModelBuilder { + private final SearchBuilder builder = new SearchBuilder(); + public TestDocumentModelBuilder addCampaign() throws ParseException { + builder.importString(joinLines("search campaign {", + " document campaign {", + " field cool_field type string { indexing: attribute }", + " field swag_field type long { indexing: attribute }", + " }", + "}")); + return this; + } + public TestDocumentModelBuilder addPerson() throws ParseException { + builder.importString(joinLines("search person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); + return this; + } + public DocumentModel build(String adSdContent) throws ParseException { + builder.importString(adSdContent); + builder.build(); + return builder.getModel(); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index e9a7a6ed33e..55980ee5fea 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -1,15 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.ReferenceDataType; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.configmodel.producers.DocumentManager; -import com.yahoo.vespa.configmodel.producers.DocumentTypes; import org.junit.Test; import java.io.IOException; @@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals; /** * @author geirst */ -public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase { +public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase { @Test public void reference_fields_can_reference_other_document_types() throws ParseException, IOException { @@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT assertEquals(campaignRefType.getTargetType(), campaignType); } - private static String TEST_FOLDER = "src/test/configmodel/types/references/"; - - private void assertDocumentConfigs(DocumentModel model, - String cfgFileSpec) throws IOException { - assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); - assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); - } - - private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { - DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); - assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); - } - - private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { - DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); - assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); - } - private static class TestDocumentModelBuilder { private final SearchBuilder builder = new SearchBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 4064e53dfb7..3dfcef70aba 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest { headerfields, new StructDataType("bodyfields"), new FieldSets(), + Collections.emptySet(), Collections.emptySet()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 68f507c810d..0f9bd506310 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -152,6 +152,14 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas " <client-authentication>need</client-authentication>", " </ssl>", " </server>", + " <server port='9003' id='with-ciphers-and-protocols'>", + " <ssl>", + " <private-key-file>/foo/key</private-key-file>", + " <certificate-file>/foo/cert</certificate-file>", + " <cipher-suites>TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384</cipher-suites>", + " <protocols>TLSv1.3</protocols>", + " </ssl>", + " </server>", " </http>", nodesXml, "", @@ -179,6 +187,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas assertThat(needClientAuth.ssl().caCertificateFile(), is(equalTo(""))); assertThat(needClientAuth.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH))); + ConnectorConfig withCiphersAndProtocols = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-ciphers-and-protocols/configured-ssl-provider@with-ciphers-and-protocols"); + assertTrue(withCiphersAndProtocols.ssl().enabled()); + assertThat(withCiphersAndProtocols.ssl().privateKeyFile(), is(equalTo("/foo/key"))); + assertThat(withCiphersAndProtocols.ssl().certificateFile(), is(equalTo("/foo/cert"))); + assertThat(withCiphersAndProtocols.ssl().enabledCipherSuites(), is(equalTo(List.of("TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384")))); + assertThat(withCiphersAndProtocols.ssl().enabledProtocols(), is(equalTo(List.of("TLSv1.3")))); + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, ConfiguredFilebasedSslProvider.COMPONENT_CLASS)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index ce36ecc4a1c..7a3b76db7f8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import com.google.common.collect.ImmutableList; import com.yahoo.config.model.ApplicationPackageTester; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; @@ -10,8 +9,10 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; @@ -35,6 +36,7 @@ public class ImportedModelTester { private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter(), new VespaImporter()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index c5c475360a3..ced7243adf5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -45,7 +45,7 @@ public class MlModelsTest { private void verify(VespaModel model) { assertEquals("Global models are created (although not used directly here", - 4, model.rankProfileList().getRankProfiles().size()); + 5, model.rankProfileList().getRankProfiles().size()); RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); model.getSearchClusters().get(0).getConfig(builder); @@ -71,8 +71,9 @@ public class MlModelsTest { "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" + + "rankingExpression(my_lightgbm).rankingScript: if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)\n" + "vespa.rank.firstphase: rankingExpression(firstphase)\n" + - "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost)\n" + + "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost) + rankingExpression(my_lightgbm)\n" + "vespa.type.attribute.argument: tensor<float>(d0[],d1[784])\n"; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index 3d4ac7f2eeb..2d3ddc33afb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -96,9 +96,10 @@ public class ModelEvaluationTest { cluster.getConfig(cb); RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb); - assertEquals(4, config.rankprofile().size()); + assertEquals(5, config.rankprofile().size()); Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet()); assertTrue(modelNames.contains("xgboost_2_2")); + assertTrue(modelNames.contains("lightgbm_regression")); assertTrue(modelNames.contains("mnist_saved")); assertTrue(modelNames.contains("mnist_softmax")); assertTrue(modelNames.contains("mnist_softmax_saved")); @@ -112,13 +113,18 @@ public class ModelEvaluationTest { ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null)) .importFrom(config, constantsConfig)); - assertEquals(4, evaluator.models().size()); + assertEquals(5, evaluator.models().size()); Model xgboost = evaluator.models().get("xgboost_2_2"); assertNotNull(xgboost); assertNotNull(xgboost.evaluatorOf()); assertNotNull(xgboost.evaluatorOf("xgboost_2_2")); + Model lightgbm = evaluator.models().get("lightgbm_regression"); + assertNotNull(lightgbm); + assertNotNull(lightgbm.evaluatorOf()); + assertNotNull(lightgbm.evaluatorOf("lightgbm_regression")); + Model tensorflow_mnist = evaluator.models().get("mnist_saved"); assertNotNull(tensorflow_mnist); assertEquals(1, tensorflow_mnist.functions().size()); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 2bbd98f72ac..1bf42650123 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -119,6 +119,13 @@ <certificate-file>/foo/cert</certificate-file> <ca-certificates-file>/foo/cacerts</ca-certificates-file> <client-authentication>want</client-authentication> + <cipher-suites> + TLS_AES_128_GCM_SHA256, + TLS_AES_256_GCM_SHA384, + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + </cipher-suites> + <protocols>TLSv1.2,TLSv1.3</protocols> </ssl> </server> <server port="4083" id="sslProvider"> |