diff options
| author | Jon Marius Venstad <jonmv@users.noreply.github.com> | 2019-01-13 15:24:00 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-01-13 15:24:00 +0100 |
| commit | 44c89edf64fcae684ab39c42b59fe8b22183f173 (patch) | |
| tree | 6bfe2ca36265bcd642106e77e37e2c0335b565da | |
| parent | 03a344eba3265b5fc5d99d849e9d52ba05a31832 (diff) | |
| parent | 028fd60d61854d074d2d8e5a4fb8b416abc7a62c (diff) | |
Merge branch 'master' into jvenstad/remove-feature-flag-for-cache-invalidation-strategy
234 files changed, 3876 insertions, 2511 deletions
diff --git a/config-lib/src/main/java/com/yahoo/config/FileReference.java b/config-lib/src/main/java/com/yahoo/config/FileReference.java index 7d455c58b30..3b95c2fbd4c 100755 --- a/config-lib/src/main/java/com/yahoo/config/FileReference.java +++ b/config-lib/src/main/java/com/yahoo/config/FileReference.java @@ -44,7 +44,7 @@ public final class FileReference { } public static List<String> toValues(Collection<FileReference> references) { - List<String> ret = new ArrayList<String>(); + List<String> ret = new ArrayList<>(); for (FileReference r: references) { ret.add(r.value()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index d94d072e75c..bc49c40e4e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -7,6 +7,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.ranking.Diversity; +import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.expressiontransforms.ExpressionTransforms; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; @@ -773,9 +774,10 @@ public class RankProfile implements Serializable, Cloneable { } private void addAttributeFeatureTypes(ImmutableSDField field, MapEvaluationTypeContext context) { + Attribute attribute = field.getAttribute(); field.getAttributes().forEach((k, a) -> { String name = k; - if (k.equals(field.getBackingField().getName())) // this attribute should take the fields name + if (attribute == a) // this attribute should take the fields name name = field.getName(); // switch to that - it is separate for imported fields context.setType(FeatureNames.asAttributeFeature(name), a.tensorType().orElse(TensorType.empty)); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index 7fd996edc39..ba2421bf5bb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -179,7 +179,7 @@ public class Search implements Serializable, ImmutableSearch { return importedFields .map(fields -> fields.fields().values().stream()) .orElse(Stream.empty()) - .map(ImmutableImportedSDField::new); + .map(field -> field.asImmutableSDField()); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 9ea17dab2ba..76dff404568 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -53,9 +53,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce if (unsupportedFieldType(field)) { return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) } - if (field.isImportedField()) { - deriveImportedAttributes(field); - } else if (isArrayOfSimpleStruct(field)) { + if (isArrayOfSimpleStruct(field)) { deriveArrayOfSimpleStruct(field); } else if (isMapOfSimpleStruct(field)) { deriveMapOfSimpleStruct(field); @@ -84,6 +82,10 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce /** Derives one attribute. TODO: Support non-default named attributes */ private void deriveAttributes(ImmutableSDField field) { + if (field.isImportedField()) { + deriveImportedAttributes(field); + return; + } for (Attribute fieldAttribute : field.getAttributes().values()) { deriveAttribute(field, fieldAttribute); } @@ -125,6 +127,10 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } private void deriveAttributeAsArrayType(ImmutableSDField field) { + if (field.isImportedField()) { + deriveImportedAttributes(field); + return; + } Attribute attribute = field.getAttributes().get(field.getName()); if (attribute != null) { attributes.put(attribute.getName(), attribute.convertToArray()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java index 82b56f9c961..b44522e9771 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java @@ -1,14 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.ImportedComplexField; import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.vespa.config.search.ImportedFieldsConfig; import java.util.Optional; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; + /** * This class derives imported fields from search definition and produces imported-fields.cfg as needed by the search backend. * @@ -44,6 +51,37 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod } private static void considerField(ImportedFieldsConfig.Builder builder, ImportedField field) { + if (field instanceof ImportedComplexField) { + considerComplexField(builder, (ImportedComplexField) field); + } else { + considerSimpleField(builder, field); + } + } + + private static void considerComplexField(ImportedFieldsConfig.Builder builder, ImportedComplexField field) { + ImmutableSDField targetField = field.targetField(); + if (targetField.getDataType().equals(PositionDataType.INSTANCE) || + targetField.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { + + } else if (isArrayOfSimpleStruct(targetField)) { + considerNestedFields(builder, field); + } else if (isMapOfSimpleStruct(targetField)) { + considerSimpleField(builder, field.getNestedField("key")); + considerNestedFields(builder, field.getNestedField("value")); + } else if (isMapOfPrimitiveType(targetField)) { + considerSimpleField(builder, field.getNestedField("key")); + considerSimpleField(builder, field.getNestedField("value")); + } + } + + private static void considerNestedFields(ImportedFieldsConfig.Builder builder, ImportedField field) { + if (field instanceof ImportedComplexField) { + ImportedComplexField complexField = (ImportedComplexField) field; + complexField.getNestedFields().forEach(nestedField -> considerSimpleField(builder, nestedField)); + } + } + + private static void considerSimpleField(ImportedFieldsConfig.Builder builder, ImportedField field) { ImmutableSDField targetField = field.targetField(); String targetFieldName = targetField.getName(); if (!isNestedFieldName(targetFieldName)) { @@ -51,7 +89,7 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod builder.attribute.add(createAttributeBuilder(field)); } } else { - Attribute attribute = targetField.getAttributes().get(targetFieldName); + Attribute attribute = targetField.getAttribute(); if (attribute != null) { builder.attribute.add(createAttributeBuilder(field)); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 4f71bd90830..1e4a4a2a5d2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -74,9 +74,6 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT); } } - search.importedFields().map(fields -> fields.complexFields().values().stream()). - orElse(Stream.empty()). - forEach(field -> deriveImportedComplexField(field)); } @Override @@ -86,19 +83,6 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } } - private void deriveImportedComplexField(ImportedField field) { - String fieldName = field.fieldName(); - if (isPositionField(field.targetField())) { - addIndexCommand(fieldName, CMD_DEFAULT_POSITION); - if (isPositionArrayField(field.targetField())) { - addIndexCommand(fieldName, CMD_MULTIVALUE); - } - } else { - addIndexCommand(fieldName, CMD_MULTIVALUE); - } - addIndexCommand(fieldName, CMD_INDEX); - } - private String toSpaceSeparated(Collection c) { StringBuffer b = new StringBuffer(); for (Iterator i = c.iterator(); i.hasNext();) { @@ -163,7 +147,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { addIndexCommand(field, CMD_MULTIVALUE); } - Attribute attribute = getAttribute(field); + Attribute attribute = field.getAttribute(); if ((field.doesAttributing() || (attribute != null && !inPosition)) && !field.doesIndexing()) { addIndexCommand(field.getName(), CMD_ATTRIBUTE); if (attribute != null && attribute.isFastSearch()) @@ -195,13 +179,6 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } - private static Attribute getAttribute(ImmutableSDField field) { - while (field.isImportedField()) { - field = field.getBackingField(); - } - return field.getAttributes().get(field.getName()); - } - static String stemCmd(ImmutableSDField field, Search search) { return CMD_STEM + ":" + field.getStemming(search).toStemMode(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java new file mode 100644 index 00000000000..2f13c0078ee --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java @@ -0,0 +1,29 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.util.Collection; + +import static java.util.stream.Collectors.toList; + +/** + * Wraps {@link ImportedComplexField} as {@link ImmutableSDField}. + */ +public class ImmutableImportedComplexSDField extends ImmutableImportedSDField { + private final ImportedComplexField importedComplexField; + + public ImmutableImportedComplexSDField(ImportedComplexField importedField) { + super(importedField); + importedComplexField = importedField; + } + + @Override + public ImmutableSDField getStructField(String name) { + ImportedField field = importedComplexField.getNestedField(name); + return (field != null) ? field.asImmutableSDField() : null; + } + + @Override + public Collection<? extends ImmutableSDField> getStructFields() { + return importedComplexField.getNestedFields().stream().map(field -> field.asImmutableSDField()).collect(toList()); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java index e1253d14747..be5f135f819 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java @@ -101,6 +101,9 @@ public class ImmutableImportedSDField implements ImmutableSDField { } @Override + public Attribute getAttribute() { return importedField.targetField().getAttribute(); } + + @Override public Map<String, String> getAliasToName() { return Collections.emptyMap(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java index 21ef60cf0b9..15e75ad8314 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java @@ -55,6 +55,8 @@ public interface ImmutableSDField { Map<String, Attribute> getAttributes(); + Attribute getAttribute(); + Map<String, String> getAliasToName(); ScriptExpression getIndexingScript(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java new file mode 100644 index 00000000000..56ef7527025 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java @@ -0,0 +1,49 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.searchdefinition.DocumentReference; + +import java.util.Collection; +import java.util.Map; + +/** + * A complex field that is imported from a concrete field in a referenced document type and given an alias name. + */ +public class ImportedComplexField extends ImportedField { + + private Map<String, ImportedField> nestedFields; + + public ImportedComplexField(String fieldName, DocumentReference reference, ImmutableSDField targetField) { + super(fieldName, reference, targetField); + nestedFields = new java.util.LinkedHashMap<>(0); + } + + @Override + public ImmutableSDField asImmutableSDField() { + return new ImmutableImportedComplexSDField(this); + } + + public void addNestedField(ImportedField importedField) { + String prefix = fieldName() + "."; + assert(importedField.fieldName().substring(0, prefix.length()).equals(prefix)); + String suffix = importedField.fieldName().substring(prefix.length()); + nestedFields.put(suffix, importedField); + } + + public Collection<ImportedField> getNestedFields() { + return nestedFields.values(); + } + + public ImportedField getNestedField(String name) { + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + ImportedField superField = nestedFields.get(superFieldName); + if (superField != null && superField instanceof ImportedComplexField) { + return ((ImportedComplexField)superField).getNestedField(subFieldName); + } + return null; + } + return nestedFields.get(name); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java index 1a6a24275ac..ab108213ac8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java @@ -8,7 +8,7 @@ import com.yahoo.searchdefinition.DocumentReference; * * @author geirst */ -public class ImportedField { +public abstract class ImportedField { private final String fieldName; private final DocumentReference reference; @@ -34,4 +34,5 @@ public class ImportedField { return targetField; } + public abstract ImmutableSDField asImmutableSDField(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java index 7c67fc422d4..2192a7e7bb1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java @@ -12,15 +12,12 @@ import java.util.Map; public class ImportedFields { private final Map<String, ImportedField> fields; - private final Map<String, ImportedField> complexFields; - public ImportedFields(Map<String, ImportedField> fields, Map<String, ImportedField> complexFields) { + public ImportedFields(Map<String, ImportedField> fields) { this.fields = fields; - this.complexFields = complexFields; } public Map<String, ImportedField> fields() { return Collections.unmodifiableMap(fields); } - public Map<String, ImportedField> complexFields() { return Collections.unmodifiableMap(complexFields); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java new file mode 100644 index 00000000000..63f7f99c772 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java @@ -0,0 +1,18 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.searchdefinition.DocumentReference; + +/** + * A simple field that is imported from a concrete field in a referenced document type and given an alias name. + */ +public class ImportedSimpleField extends ImportedField { + public ImportedSimpleField(String fieldName, DocumentReference reference, ImmutableSDField targetField) { + super(fieldName, reference, targetField); + } + + @Override + public ImmutableSDField asImmutableSDField() { + return new ImmutableImportedSDField(this); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 16e1e2e4e1d..049f5392c04 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -634,6 +634,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, @Override public Map<String, Attribute> getAttributes() { return attributes; } + public Attribute getAttribute() { + return attributes.get(getName()); + } + public void addAttribute(Attribute attribute) { String name = attribute.getName(); if (name == null || "".equals(name)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java index e324944549c..59dc4275e15 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -4,7 +4,9 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.ImmutableImportedComplexSDField; import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.ImportedComplexField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -30,19 +32,21 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { search.allImportedFields() - .flatMap(this::getSummaryFieldsForImportedField) - .forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); - search.importedFields().map(fields -> fields.complexFields().values().stream()). - orElse(Stream.empty()). - map(ImmutableImportedSDField::new). - flatMap(this::getSummaryFieldsForImportedField). - forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeCombinerTransform); + .forEach(field -> setTransform(field)); } private Stream<SummaryField> getSummaryFieldsForImportedField(ImmutableSDField importedField) { return search.getSummaryFields(importedField).values().stream(); } + private void setTransform(ImmutableSDField field) { + if (field instanceof ImmutableImportedComplexSDField) { + getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeCombinerTransform); + } else { + getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); + } + } + private static void setAttributeTransform(SummaryField summaryField) { if (summaryField.getTransform() == SummaryTransform.NONE) { summaryField.setTransform(SummaryTransform.ATTRIBUTE); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java index 0bc9a517d2e..e59fcdf3dd0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java @@ -42,7 +42,7 @@ public class AdjustPositionSummaryFields extends Processor { if (isPositionDataType(summaryField.getDataType())) { String originalSource = summaryField.getSingleSource(); if (originalSource.indexOf('.') == -1) { // Eliminate summary fields with pos.x or pos.y as source - ImmutableSDField sourceField = getSourceField(originalSource); + ImmutableSDField sourceField = search.getField(originalSource); if (sourceField != null) { String zCurve = null; if (sourceField.getDataType().equals(summaryField.getDataType())) { @@ -95,26 +95,12 @@ public class AdjustPositionSummaryFields extends Processor { summary.add(oldField); } - private ImmutableSDField getSourceField(String name) { - ImmutableSDField field = search.getField(name); - if (field == null && search.importedFields().isPresent()) { - ImportedField importedField = search.importedFields().get().complexFields().get(name); - if (importedField != null) { - field = new ImmutableImportedSDField(importedField); - } - } - return field; - } - private boolean hasPositionAttribute(String name) { Attribute attribute = search.getAttribute(name); if (attribute == null) { ImmutableSDField field = search.getField(name); if (field != null && field.isImportedField()) { - while (field.isImportedField()) { - field = field.getBackingField(); - } - attribute = field.getAttributes().get(field.getName()); + attribute = field.getAttribute(); } } return attribute != null && attribute.isPosition(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index c8ecc31103a..d6c334ee80b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -2,19 +2,18 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; -import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; -import com.yahoo.document.StructDataType; import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.DocumentReferences; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.ImportedComplexField; import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.searchdefinition.document.ImportedFields; +import com.yahoo.searchdefinition.document.ImportedSimpleField; import com.yahoo.searchdefinition.document.TemporaryImportedField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -34,7 +33,6 @@ import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isM public class ImportedFieldsResolver extends Processor { private final Map<String, ImportedField> importedFields = new LinkedHashMap<>(); - private final Map<String, ImportedField> importedComplexFields = new LinkedHashMap<>(); private final Optional<DocumentReferences> references; public ImportedFieldsResolver(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { @@ -45,7 +43,7 @@ public class ImportedFieldsResolver extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); - search.setImportedFields(new ImportedFields(importedFields, importedComplexFields)); + search.setImportedFields(new ImportedFields(importedFields)); } private void resolveImportedField(TemporaryImportedField importedField, boolean validate) { @@ -71,34 +69,41 @@ public class ImportedFieldsResolver extends Processor { reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); resolveImportedNormalField(importedZCurveField, reference, targetZCurveField, validate); - makeImportedComplexField(importedField, reference, targetField); + ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName(), reference, targetField); + registerImportedField(importedField, null, importedStructField); } private void resolveImportedArrayOfStructField(TemporaryImportedField importedField, DocumentReference reference, ImmutableSDField targetField, boolean validate) { - resolveImportedNestedStructField(importedField, reference, targetField, validate); - makeImportedComplexField(importedField, reference, targetField); + ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName(), reference, targetField); + resolveImportedNestedStructField(importedField, reference, importedStructField, targetField, validate); + registerImportedField(importedField, null, importedStructField); } private void resolveImportedMapOfStructField(TemporaryImportedField importedField, DocumentReference reference, ImmutableSDField targetField, boolean validate) { - resolveImportedNestedField(importedField, reference, targetField.getStructField("key"), validate); - resolveImportedNestedStructField(importedField, reference, targetField.getStructField("value"), validate); - makeImportedComplexField(importedField, reference, targetField); + ImportedComplexField importedMapField = new ImportedComplexField(importedField.fieldName(), reference, targetField); + ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName() + ".value", reference, targetField.getStructField("value")); + importedMapField.addNestedField(importedStructField); + resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("key"), validate); + resolveImportedNestedStructField(importedField, reference, importedStructField, importedStructField.targetField(), validate); + registerImportedField(importedField, null, importedMapField); } - private void makeImportedNormalField(TemporaryImportedField importedField, String name, DocumentReference reference, - ImmutableSDField targetField) { - if (importedFields.get(name) != null) { - fail(importedField, name, targetFieldAsString(targetField.getName(), reference) +": Field already imported"); - } - importedFields.put(name, new ImportedField(name, reference, targetField)); + private void makeImportedNormalField(TemporaryImportedField importedField, ImportedComplexField owner, String name, DocumentReference reference, ImmutableSDField targetField) { + ImportedField importedSimpleField = new ImportedSimpleField(name, reference, targetField); + registerImportedField(importedField, owner, importedSimpleField); } - private void makeImportedComplexField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField) { - String name = importedField.fieldName(); - importedComplexFields.put(name, new ImportedField(name, reference, targetField)); + private void registerImportedField(TemporaryImportedField temporaryImportedField, ImportedComplexField owner, ImportedField importedField) { + if (owner != null) { + owner.addNestedField(importedField); + } else { + if (importedFields.get(importedField.fieldName()) != null) { + fail(temporaryImportedField, importedField.fieldName(), targetFieldAsString(importedField.targetField().getName(), importedField.reference()) + ": Field already imported"); + } + importedFields.put(importedField.fieldName(), importedField); + } } private static String makeImportedNestedFieldName(TemporaryImportedField importedField, ImmutableSDField targetNestedField) { @@ -106,11 +111,11 @@ public class ImportedFieldsResolver extends Processor { } private boolean resolveImportedNestedField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetNestedField, boolean requireAttribute) { - Attribute attribute = targetNestedField.getAttributes().get(targetNestedField.getName()); + ImportedComplexField owner, ImmutableSDField targetNestedField, boolean requireAttribute) { + Attribute attribute = targetNestedField.getAttribute(); String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField); if (attribute != null) { - makeImportedNormalField(importedField, importedNestedFieldName, reference, targetNestedField); + makeImportedNormalField(importedField, owner, importedNestedFieldName, reference, targetNestedField); } else if (requireAttribute) { fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) + ": Is not an attribute field. Only attribute fields supported"); @@ -119,10 +124,10 @@ public class ImportedFieldsResolver extends Processor { } private void resolveImportedNestedStructField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetNestedField, boolean validate) { + ImportedComplexField ownerField, ImmutableSDField targetNestedField, boolean validate) { boolean foundAttribute = false; for (ImmutableSDField targetStructField : targetNestedField.getStructFields()) { - if (resolveImportedNestedField(importedField, reference, targetStructField, false)) { + if (resolveImportedNestedField(importedField, reference, ownerField, targetStructField, false)) { foundAttribute = true; }; } @@ -135,9 +140,10 @@ public class ImportedFieldsResolver extends Processor { private void resolveImportedMapOfPrimitiveField(TemporaryImportedField importedField, DocumentReference reference, ImmutableSDField targetField, boolean validate) { - resolveImportedNestedField(importedField, reference, targetField.getStructField("key"), validate); - resolveImportedNestedField(importedField, reference, targetField.getStructField("value"), validate); - makeImportedComplexField(importedField, reference, targetField); + ImportedComplexField importedMapField = new ImportedComplexField(importedField.fieldName(), reference, targetField); + resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("key"), validate); + resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("value"), validate); + registerImportedField(importedField, null, importedMapField); } private void resolveImportedNormalField(TemporaryImportedField importedField, DocumentReference reference, @@ -145,7 +151,7 @@ public class ImportedFieldsResolver extends Processor { if (validate) { validateTargetField(importedField, targetField, reference); } - makeImportedNormalField(importedField, importedField.fieldName(), reference, targetField); + makeImportedNormalField(importedField, null, importedField.fieldName(), reference, targetField); } private DocumentReference validateDocumentReference(TemporaryImportedField importedField) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java index 008b3182d8f..c87801685bb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java @@ -51,7 +51,6 @@ public class SummaryFieldsMustHaveValidSource extends Processor { return isDocumentField(source) || (isNotInThisSummaryClass(summary, source) && isSummaryField(source)) || (isInThisSummaryClass(summary, source) && !source.equals(summaryField.getName())) || - (search.importedFields().map(fields -> fields.complexFields().get(source) != null).orElse(false)) || (SummaryClass.DOCUMENT_ID_FIELD.equals(source)); } diff --git a/config-model/src/test/derived/imported_position_field_summary/index-info.cfg b/config-model/src/test/derived/imported_position_field_summary/index-info.cfg index c87553d6537..4c8dafdf59b 100644 --- a/config-model/src/test/derived/imported_position_field_summary/index-info.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/index-info.cfg @@ -9,18 +9,6 @@ indexinfo[].command[].indexname "parent_ref" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "parent_ref" indexinfo[].command[].command "word" -indexinfo[].command[].indexname "my_pos.x" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "my_pos.x" -indexinfo[].command[].command "numerical" -indexinfo[].command[].indexname "my_pos.y" -indexinfo[].command[].command "index" -indexinfo[].command[].indexname "my_pos.y" -indexinfo[].command[].command "numerical" -indexinfo[].command[].indexname "my_pos" -indexinfo[].command[].command "default-position" -indexinfo[].command[].indexname "my_pos" -indexinfo[].command[].command "index" indexinfo[].command[].indexname "my_pos.distance" indexinfo[].command[].command "index" indexinfo[].command[].indexname "my_pos.distance" @@ -41,3 +29,7 @@ indexinfo[].command[].indexname "my_pos_zcurve" indexinfo[].command[].command "fast-search" indexinfo[].command[].indexname "my_pos_zcurve" indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "my_pos" +indexinfo[].command[].command "default-position" +indexinfo[].command[].indexname "my_pos" +indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/imported_struct_fields/index-info.cfg b/config-model/src/test/derived/imported_struct_fields/index-info.cfg index b32b487fe0f..8d7f4d4bece 100644 --- a/config-model/src/test/derived/imported_struct_fields/index-info.cfg +++ b/config-model/src/test/derived/imported_struct_fields/index-info.cfg @@ -1,71 +1,73 @@ -indexinfo[0].name "child" -indexinfo[0].command[0].indexname "sddocname" -indexinfo[0].command[0].command "index" -indexinfo[0].command[1].indexname "sddocname" -indexinfo[0].command[1].command "word" -indexinfo[0].command[2].indexname "parent_ref" -indexinfo[0].command[2].command "index" -indexinfo[0].command[3].indexname "parent_ref" -indexinfo[0].command[3].command "attribute" -indexinfo[0].command[4].indexname "parent_ref" -indexinfo[0].command[4].command "word" -indexinfo[0].command[5].indexname "documentid" -indexinfo[0].command[5].command "index" -indexinfo[0].command[6].indexname "rankfeatures" -indexinfo[0].command[6].command "index" -indexinfo[0].command[7].indexname "summaryfeatures" -indexinfo[0].command[7].command "index" -indexinfo[0].command[8].indexname "my_elem_array.name" -indexinfo[0].command[8].command "index" -indexinfo[0].command[9].indexname "my_elem_array.name" -indexinfo[0].command[9].command "attribute" -indexinfo[0].command[10].indexname "my_elem_array.name" -indexinfo[0].command[10].command "fast-search" -indexinfo[0].command[11].indexname "my_elem_array.weight" -indexinfo[0].command[11].command "index" -indexinfo[0].command[12].indexname "my_elem_array.weight" -indexinfo[0].command[12].command "attribute" -indexinfo[0].command[13].indexname "my_elem_array.weight" -indexinfo[0].command[13].command "numerical" -indexinfo[0].command[14].indexname "my_elem_map.key" -indexinfo[0].command[14].command "index" -indexinfo[0].command[15].indexname "my_elem_map.key" -indexinfo[0].command[15].command "attribute" -indexinfo[0].command[16].indexname "my_elem_map.key" -indexinfo[0].command[16].command "fast-search" -indexinfo[0].command[17].indexname "my_elem_map.value.name" -indexinfo[0].command[17].command "index" -indexinfo[0].command[18].indexname "my_elem_map.value.name" -indexinfo[0].command[18].command "attribute" -indexinfo[0].command[19].indexname "my_elem_map.value.name" -indexinfo[0].command[19].command "fast-search" -indexinfo[0].command[20].indexname "my_elem_map.value.weight" -indexinfo[0].command[20].command "index" -indexinfo[0].command[21].indexname "my_elem_map.value.weight" -indexinfo[0].command[21].command "attribute" -indexinfo[0].command[22].indexname "my_elem_map.value.weight" -indexinfo[0].command[22].command "numerical" -indexinfo[0].command[23].indexname "my_str_int_map.key" -indexinfo[0].command[23].command "index" -indexinfo[0].command[24].indexname "my_str_int_map.key" -indexinfo[0].command[24].command "attribute" -indexinfo[0].command[25].indexname "my_str_int_map.key" -indexinfo[0].command[25].command "fast-search" -indexinfo[0].command[26].indexname "my_str_int_map.value" -indexinfo[0].command[26].command "index" -indexinfo[0].command[27].indexname "my_str_int_map.value" -indexinfo[0].command[27].command "attribute" -indexinfo[0].command[28].indexname "my_str_int_map.value" -indexinfo[0].command[28].command "numerical" -indexinfo[0].command[29].indexname "my_elem_array" -indexinfo[0].command[29].command "multivalue" -indexinfo[0].command[30].indexname "my_elem_array" -indexinfo[0].command[30].command "index" -indexinfo[0].command[31].indexname "my_elem_map" -indexinfo[0].command[31].command "multivalue" -indexinfo[0].command[32].indexname "my_elem_map" -indexinfo[0].command[32].command "index" -indexinfo[0].command[33].indexname "my_str_int_map" -indexinfo[0].command[33].command "multivalue" -indexinfo[0].command[34].indexname "my_str_int_map" -indexinfo[0].command[34].command "index"
\ No newline at end of file +indexinfo[].name "child" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "parent_ref" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "parent_ref" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "parent_ref" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "documentid" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "rankfeatures" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "summaryfeatures" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_array.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_array.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "my_elem_array.name" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "my_elem_array.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_array.weight" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "my_elem_array.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "my_elem_array" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_array" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_map.value.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_map.value.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "my_elem_map.value.name" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "my_elem_map.value.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_map.value.weight" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "my_elem_map.value.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "my_elem_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_map.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "my_elem_map.key" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "my_elem_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_elem_map" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_str_int_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_str_int_map.key" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "my_str_int_map.key" +indexinfo[].command[].command "fast-search" +indexinfo[].command[].indexname "my_str_int_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_str_int_map.value" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "my_str_int_map.value" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "my_str_int_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "my_str_int_map" +indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index 3735b997073..defaf565a8b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -8,6 +8,7 @@ import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.searchdefinition.document.ImportedFields; +import com.yahoo.searchdefinition.document.ImportedSimpleField; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; @@ -53,8 +54,8 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { Search targetSearch = new Search("target_doc", MockApplicationPackage.createEmpty()); SDField targetField = new SDField("target_field", DataType.INT); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSearch); - ImportedField importedField = new ImportedField(fieldName, documentReference, targetField); - return new ImportedFields(Collections.singletonMap(fieldName, importedField), Collections.emptyMap()); + ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); + return new ImportedFields(Collections.singletonMap(fieldName, importedField)); } private static DocumentSummary createDocumentSummary(String fieldName) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java index 48b79dade1f..724c15c3ef4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.document.ImportedComplexField; import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Rule; @@ -91,7 +92,7 @@ public class ImportedFieldsTestCase { " import field parent_ref.elem_map as my_elem_map {}", " import field parent_ref.str_int_map as my_str_int_map {}", "}")); - assertEquals(parentBuilder.countAttrs(), search.importedFields().get().fields().size()); + assertEquals(3, search.importedFields().get().fields().size()); checkImportedField("my_elem_array.name", "parent_ref", "parent", "elem_array.name", search, parentBuilder.elem_array_name_attr); checkImportedField("my_elem_array.weight", "parent_ref", "parent", "elem_array.weight", search, parentBuilder.elem_array_weight_attr); checkImportedField("my_elem_map.key", "parent_ref", "parent", "elem_map.key", search, parentBuilder.elem_map_key_attr); @@ -99,6 +100,9 @@ public class ImportedFieldsTestCase { checkImportedField("my_elem_map.value.weight", "parent_ref", "parent", "elem_map.value.weight", search, parentBuilder.elem_map_value_weight_attr); checkImportedField("my_str_int_map.key", "parent_ref", "parent", "str_int_map.key", search, parentBuilder.str_int_map_key_attr); checkImportedField("my_str_int_map.value", "parent_ref", "parent", "str_int_map.value", search, parentBuilder.str_int_map_value_attr); + checkImportedField("my_elem_array", "parent_ref", "parent", "elem_array", search, true); + checkImportedField("my_elem_map", "parent_ref", "parent", "elem_map", search, true); + checkImportedField("my_str_int_map", "parent_ref", "parent", "str_int_map", search, true); } @Test @@ -275,8 +279,9 @@ public class ImportedFieldsTestCase { private static void checkPosImport(ParentPosSdBuilder parentBuilder, ChildPosSdBuilder childBuilder) throws ParseException { Search search = buildChildSearch(parentBuilder.build(), childBuilder.build()); - assertEquals(1, search.importedFields().get().fields().size()); + assertEquals(2, search.importedFields().get().fields().size()); assertSearchContainsImportedField("my_pos_zcurve", "parent_ref", "parent", "pos_zcurve", search); + assertSearchContainsImportedField("my_pos", "parent_ref", "parent", "pos", search); } @Test @@ -291,8 +296,22 @@ public class ImportedFieldsTestCase { checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos_zcurve_before(true)); } + private static ImportedField getImportedField(String name, Search search) { + if (name.contains(".")) { + assertNull(search.importedFields().get().fields().get(name)); + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + ImportedField superField = search.importedFields().get().fields().get(superFieldName); + if (superField != null && superField instanceof ImportedComplexField) { + return ((ImportedComplexField)superField).getNestedField(subFieldName); + } + return null; + } + return search.importedFields().get().fields().get(name); + } + private static void assertSearchNotContainsImportedField(String fieldName, Search search) { - ImportedField importedField = search.importedFields().get().fields().get(fieldName); + ImportedField importedField = getImportedField(fieldName, search); assertNull(importedField); } @@ -301,7 +320,7 @@ public class ImportedFieldsTestCase { String referenceDocType, String targetFieldName, Search search) { - ImportedField importedField = search.importedFields().get().fields().get(fieldName); + ImportedField importedField = getImportedField(fieldName, search); assertNotNull(importedField); assertEquals(fieldName, importedField.fieldName()); assertEquals(referenceFieldName, importedField.reference().referenceField().getName()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index e4c23f407c8..cec313f98d8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -8,6 +8,7 @@ import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.searchdefinition.document.ImportedFields; +import com.yahoo.searchdefinition.document.ImportedSimpleField; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; @@ -54,8 +55,8 @@ public class ValidateFieldTypesTest { Search targetSearch = new Search("target_doc", MockApplicationPackage.createEmpty()); SDField targetField = new SDField("target_field", dataType); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSearch); - ImportedField importedField = new ImportedField(fieldName, documentReference, targetField); - return new ImportedFields(Collections.singletonMap(fieldName, importedField), Collections.emptyMap()); + ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); + return new ImportedFields(Collections.singletonMap(fieldName, importedField)); } private static DocumentSummary createDocumentSummary(String fieldName, DataType dataType) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 6390c9ca165..6274ec77e01 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -89,6 +89,7 @@ public class ProxyServer implements Runnable { this.configClient = createClient(clientUpdater, delayedResponses, source, timingValues, memoryCache, configClient); this.fileDownloader = new FileDownloader(new JRTConnectionPool(source)); new FileDistributionRpcServer(supervisor, fileDownloader); + new UrlDownloadRpcServer(supervisor); } static ProxyServer createTestServer(ConfigSourceSet source) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java new file mode 100644 index 00000000000..d8688d5cc36 --- /dev/null +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java @@ -0,0 +1,148 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.proxy; + +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.defaults.Defaults; +import net.jpountz.xxhash.XXHashFactory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Logger; + +import static com.yahoo.vespa.config.UrlDownloader.DOES_NOT_EXIST; +import static com.yahoo.vespa.config.UrlDownloader.HTTP_ERROR; +import static com.yahoo.vespa.config.UrlDownloader.INTERNAL_ERROR; + +/** + * An RPC server that handles URL download requests. + * + * @author lesters + */ +public class UrlDownloadRpcServer { + private final static Logger log = Logger.getLogger(UrlDownloadRpcServer.class.getName()); + + private static final String CONTENTS_FILE_NAME = "contents"; + private static final String LAST_MODFIED_FILE_NAME = "lastmodified"; + + private final File downloadBaseDir; + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + new DaemonThreadFactory("Rpc download executor")); + + public UrlDownloadRpcServer(Supervisor supervisor) { + supervisor.addMethod(new Method("url.waitFor", "s", "s", this, "download") + .methodDesc("get path to url download") + .paramDesc(0, "url", "url") + .returnDesc(0, "path", "path to file")); + downloadBaseDir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/download")); + } + + @SuppressWarnings({"UnusedDeclaration"}) + public final void download(Request req) { + req.detach(); + rpcDownloadExecutor.execute(() -> downloadFile(req)); + } + + private void downloadFile(Request req) { + String url = req.parameters().get(0).asString(); + File downloadDir = new File(this.downloadBaseDir, urlToDirName(url)); + + try { + URL website = new URL(url); + HttpURLConnection connection = (HttpURLConnection) website.openConnection(); + setIfModifiedSince(connection, downloadDir); // don't download if we already have the file + + if (connection.getResponseCode() == 200) { + log.log(LogLevel.INFO, "Downloading URL '" + url + "'"); + downloadFile(req, connection, downloadDir); + + } else if (connection.getResponseCode() == 304) { + log.log(LogLevel.INFO, "URL '" + url + "' already downloaded (server response: 304)"); + req.returnValues().add(new StringValue(new File(downloadDir, CONTENTS_FILE_NAME).getAbsolutePath())); + + } else { + log.log(LogLevel.ERROR, "Download of URL '" + url + "' got server response: " + connection.getResponseCode()); + req.setError(HTTP_ERROR, String.valueOf(connection.getResponseCode())); + } + + } catch (Throwable e) { + log.log(LogLevel.ERROR, "Download of URL '" + url + "' got exception: " + e.getMessage()); + req.setError(INTERNAL_ERROR, "Download of URL '" + url + "' internal error: " + e.getMessage()); + } + req.returnRequest(); + } + + private static void downloadFile(Request req, HttpURLConnection connection, File downloadDir) throws IOException { + long start = System.currentTimeMillis(); + String url = connection.getURL().toString(); + Files.createDirectories(downloadDir.toPath()); + File contentsPath = new File(downloadDir, CONTENTS_FILE_NAME); + try (ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream())) { + try (FileOutputStream fos = new FileOutputStream((contentsPath.getAbsolutePath()))) { + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + + if (contentsPath.exists() && contentsPath.length() > 0) { + writeLastModifiedTimestamp(downloadDir, connection.getLastModified()); + req.returnValues().add(new StringValue(contentsPath.getAbsolutePath())); + log.log(LogLevel.DEBUG, () -> "URL '" + url + "' available at " + contentsPath); + } else { + log.log(LogLevel.ERROR, "Downloaded URL '" + url + "' not found, returning error"); + req.setError(DOES_NOT_EXIST, "Downloaded '" + url + "' not found"); + } + } + } + long end = System.currentTimeMillis(); + log.log(LogLevel.INFO, String.format("Download of URL '%s' done in %.3f seconds", url, (end-start) / 1000.0)); + } + + private static String urlToDirName(String uri) { + return String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); + } + + private static void setIfModifiedSince(HttpURLConnection connection, File downloadDir) throws IOException { + File contents = new File(downloadDir, CONTENTS_FILE_NAME); + if (contents.exists() && contents.length() > 0) { + long lastModified = readLastModifiedTimestamp(downloadDir); + if (lastModified > 0) { + connection.setIfModifiedSince(lastModified); + } + } + } + + private static long readLastModifiedTimestamp(File downloadDir) throws IOException { + File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME); + if (lastModified.exists() && lastModified.length() > 0) { + try (BufferedReader br = new BufferedReader(new FileReader(lastModified))) { + String timestamp = br.readLine(); + return Long.parseLong(timestamp); + } + } + return 0; + } + + private static void writeLastModifiedTimestamp(File downloadDir, long timestamp) throws IOException { + File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME); + try (BufferedWriter lastModifiedWriter = new BufferedWriter(new FileWriter(lastModified.getAbsolutePath()))) { + lastModifiedWriter.write(Long.toString(timestamp)); + } + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 8a12405d505..40d79afc854 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -20,7 +20,11 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.nio.file.Path; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Stack; import java.util.logging.Logger; /** @@ -36,15 +40,17 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private final ConfigInstance.Builder rootBuilder; private final ConfigTransformer.PathAcquirer pathAcquirer; + private final UrlDownloader urlDownloader; private final Stack<NamedBuilder> stack = new Stack<>(); public ConfigPayloadApplier(T builder) { - this(builder, new IdentityPathAcquirer()); + this(builder, new IdentityPathAcquirer(), null); } - public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer) { + public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer, UrlDownloader urlDownloader) { this.rootBuilder = builder; this.pathAcquirer = pathAcquirer; + this.urlDownloader = urlDownloader; debug("rootBuilder=" + rootBuilder); } @@ -207,6 +213,12 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { if (isPathField(builder, methodName)) { FileReference wrappedPath = resolvePath((String)value); invokeSetter(builder, methodName, key, wrappedPath); + + // Need to convert url into actual file if 'url' type is used + } else if (isUrlField(builder, methodName)) { + UrlReference url = resolveUrl((String)value); + invokeSetter(builder, methodName, key, url); + } else { invokeSetter(builder, methodName, key, value); } @@ -258,7 +270,8 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { // Need to convert url into actual file if 'url' type is used } else if (isUrlField(builder, methodName)) { - throw new UnsupportedOperationException("'url' type is not yet implemented"); + UrlReference url = resolveUrl(Utf8.toString(value.asUtf8())); + invokeSetter(builder, methodName, url); } else { Object object = getValueFromInspector(value); @@ -276,6 +289,14 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return newFileReference(path.toString()); } + private UrlReference resolveUrl(String url) { + if (urlDownloader == null || !urlDownloader.isValid()) { + throw new RuntimeException("Resolving url field failed due to missing or invalid URL downloader."); + } + File file = urlDownloader.waitFor(new UrlReference(url), 60 * 60); + return new UrlReference(file.getAbsolutePath()); + } + private FileReference newFileReference(String fileReference) { try { Constructor<FileReference> constructor = FileReference.class.getDeclaredConstructor(String.class); @@ -343,18 +364,19 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { * Checks whether or not this field is of type 'path', in which * case some special handling might be needed. Caches the result. */ + private Set<String> pathFieldSet = new HashSet<>(); private boolean isPathField(Object builder, String methodName) { // Paths are stored as FileReference in Builder. - return isFieldType(builder, methodName, FileReference.class); + return isFieldType(pathFieldSet, builder, methodName, FileReference.class); } + private Set<String> urlFieldSet = new HashSet<>(); private boolean isUrlField(Object builder, String methodName) { // Urls are stored as UrlReference in Builder. - return isFieldType(builder, methodName, UrlReference.class); + return isFieldType(urlFieldSet, builder, methodName, UrlReference.class); } - private Set<String> fieldSet = new HashSet<>(); - private boolean isFieldType(Object builder, String methodName, java.lang.reflect.Type type) { + private boolean isFieldType(Set<String> fieldSet, Object builder, String methodName, java.lang.reflect.Type type) { String key = fieldKey(builder, methodName); if (fieldSet.contains(key)) { return true; @@ -515,4 +537,5 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return new File(fileReference.value()).toPath(); } } + } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java index ac3f490182a..163e010cdd6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -26,6 +26,7 @@ public class ConfigTransformer<T extends ConfigInstance> { private final Class<T> clazz; private static volatile PathAcquirer pathAcquirer = new IdentityPathAcquirer(); + private static volatile UrlDownloader urlDownloader; /** * For internal use only * @@ -36,6 +37,10 @@ public class ConfigTransformer<T extends ConfigInstance> { pathAcquirer; } + public static void setUrlDownloader(UrlDownloader urlDownloader) { + ConfigTransformer.urlDownloader = urlDownloader; + } + /** * Create a transformer capable of converting payloads to clazz * @@ -53,7 +58,7 @@ public class ConfigTransformer<T extends ConfigInstance> { */ public ConfigInstance.Builder toConfigBuilder(ConfigPayload payload) { ConfigInstance.Builder builder = getRootBuilder(); - ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer); + ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer, urlDownloader); creator.applyPayload(payload); return builder; } diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java new file mode 100644 index 00000000000..4947b618f50 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -0,0 +1,98 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.UrlReference; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.util.logging.Logger; + +/** + * @author lesters + */ +public class UrlDownloader { + + private static final Logger log = Logger.getLogger(UrlDownloader.class.getName()); + + private static final int BASE_ERROR_CODE = 0x10000; + public static final int DOES_NOT_EXIST = BASE_ERROR_CODE + 1; + public static final int INTERNAL_ERROR = BASE_ERROR_CODE + 2; + public static final int HTTP_ERROR = BASE_ERROR_CODE + 3; + + private final Supervisor supervisor = new Supervisor(new Transport()); + private final Spec spec; + private Target target; + + public UrlDownloader() { + spec = new Spec(Defaults.getDefaults().vespaHostname(), Defaults.getDefaults().vespaConfigProxyRpcPort()); + connect(); + } + + public void shutdown() { + supervisor.transport().shutdown().join(); + } + + private void connect() { + int timeRemaining = 5000; + try { + while (timeRemaining > 0) { + target = supervisor.connectSync(spec); + if (target.isValid()) { + log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + return; + } + Thread.sleep(500); + timeRemaining -= 500; + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public boolean isValid() { + return target.isValid(); + } + + private boolean temporaryError(Request req) { + return false; // Currently, none of the errors are considered temporary + } + + public File waitFor(UrlReference urlReference, long timeout) { + long start = System.currentTimeMillis() / 1000; + long timeLeft = timeout; + do { + Request request = new Request("url.waitFor"); + request.parameters().add(new StringValue(urlReference.value())); + + double rpcTimeout = Math.min(timeLeft, 60 * 60.0); + log.log(LogLevel.DEBUG, "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + target.invokeSync(request, rpcTimeout); + + if (request.checkReturnTypes("s")) { + return new File(request.returnValues().get(0).asString()); + } else if (!request.isError()) { + throw new RuntimeException("Invalid response: " + request.returnValues()); + } else if (temporaryError(request)) { + log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted sleep between retries of waitFor", e); + } + } else { + throw new RuntimeException("Wait for " + urlReference + " failed: " + request.errorMessage() + " (" + request.errorCode() + ")"); + } + timeLeft = start + timeout - System.currentTimeMillis() / 1000; + } while (timeLeft > 0); + + throw new RuntimeException("Timed out waiting for " + urlReference + " after " + timeout); + } + +} diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java index 9768c42b477..d452d8a4aad 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.configserver.flags; import com.google.inject.Inject; -import com.yahoo.vespa.configserver.flags.db.FlagsDbImpl; import com.yahoo.vespa.configserver.flags.db.ZooKeeperFlagSource; import com.yahoo.vespa.flags.FileFlagSource; import com.yahoo.vespa.flags.OrderedFlagSource; @@ -12,7 +11,7 @@ import com.yahoo.vespa.flags.OrderedFlagSource; */ public class ConfigServerFlagSource extends OrderedFlagSource { @Inject - public ConfigServerFlagSource(FlagsDbImpl flagsDb) { + public ConfigServerFlagSource(FlagsDb flagsDb) { super(new FileFlagSource(), new ZooKeeperFlagSource(flagsDb)); } } diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java index bd99ac6eca9..4a9d604b4bd 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.configserver.flags.db; +import com.yahoo.vespa.configserver.flags.FlagsDb; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.FlagSource; @@ -12,9 +13,9 @@ import java.util.Optional; * @author hakonhall */ public class ZooKeeperFlagSource implements FlagSource { - private final FlagsDbImpl flagsDb; + private final FlagsDb flagsDb; - public ZooKeeperFlagSource(FlagsDbImpl flagsDb) { + public ZooKeeperFlagSource(FlagsDb flagsDb) { this.flagsDb = flagsDb; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 2a8abacd24e..f710eec27ba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -23,6 +23,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.logging.Logger; public class FileDirectory { + private static final Logger log = Logger.getLogger(FileDirectory.class.getName()); private final File root; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataListResponse.java index ae72660ce9f..b33fc7c2b04 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagDataListResponse.java @@ -9,6 +9,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.http.HttpConfigResponse; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagDataList; import java.io.OutputStream; import java.util.Map; @@ -23,31 +24,32 @@ public class FlagDataListResponse extends HttpResponse { private static ObjectMapper mapper = new ObjectMapper(); private final String flagsV1Uri; - private final Map<FlagId, FlagData> flags; + private final TreeMap<FlagId, FlagData> flags; private final boolean recursive; public FlagDataListResponse(String flagsV1Uri, Map<FlagId, FlagData> flags, boolean recursive) { super(Response.Status.OK); this.flagsV1Uri = flagsV1Uri; - this.flags = flags; + this.flags = new TreeMap<>(flags); this.recursive = recursive; } @Override public void render(OutputStream outputStream) { - ObjectNode rootNode = mapper.createObjectNode(); - ArrayNode flagsArray = rootNode.putArray("flags"); - // Order flags by ID - new TreeMap<>(this.flags).forEach((flagId, flagData) -> { - if (recursive) { - flagsArray.add(flagData.toJsonNode()); - } else { + if (recursive) { + WireFlagDataList list = new WireFlagDataList(); + flags.values().forEach(flagData -> list.flags.add(flagData.toWire())); + list.serializeToOutputStream(outputStream); + } else { + ObjectNode rootNode = mapper.createObjectNode(); + ArrayNode flagsArray = rootNode.putArray("flags"); + flags.forEach((flagId, flagData) -> { ObjectNode object = flagsArray.addObject(); object.put("id", flagId.toString()); object.put("url", flagsV1Uri + "/data/" + flagId.toString()); - } - }); - uncheck(() -> mapper.writeValue(outputStream, rootNode)); + }); + uncheck(() -> mapper.writeValue(outputStream, rootNode)); + } } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagsHandler.java index e5efc74fb75..1b1595eb897 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/flags/FlagsHandler.java @@ -64,7 +64,8 @@ public class FlagsHandler extends HttpHandler { private String flagsV1Uri(HttpRequest request) { URI uri = request.getUri(); - return uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + "/flags/v1"; + String port = uri.getPort() < 0 ? "" : ":" + uri.getPort(); + return uri.getScheme() + "://" + uri.getHost() + port + "/flags/v1"; } private HttpResponse getFlagDataList(HttpRequest request) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java index 5813c3eb04a..87e9fa287a4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java @@ -48,8 +48,8 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000); - timer.scheduleAtFixedRate(this, 5000, intervalMs); - zkMetricUpdater = new ZKMetricUpdater(zkServerConfig, 4500, intervalMs); + timer.scheduleAtFixedRate(this, 20000, intervalMs); + zkMetricUpdater = new ZKMetricUpdater(zkServerConfig, 19500, intervalMs); } public static Metrics createTestMetrics() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java index d61f8a7b9fe..408bf44e733 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java @@ -43,10 +43,10 @@ public class ZKMetricUpdater extends TimerTask { private final Timer timer = new Timer(); private final int zkPort; - public ZKMetricUpdater(ZookeeperServerConfig zkServerConfig, long delay, long interval) { + public ZKMetricUpdater(ZookeeperServerConfig zkServerConfig, long delayMS, long intervalMS) { this.zkPort = zkServerConfig.clientPort(); - if (interval > 0) { - timer.scheduleAtFixedRate(this, delay, interval); + if (intervalMS > 0) { + timer.scheduleAtFixedRate(this, delayMS, intervalMS); } } diff --git a/configserver/src/main/resources/logd/logd.cfg b/configserver/src/main/resources/logd/logd.cfg index 137f3742ef2..a422a185dd5 100644 --- a/configserver/src/main/resources/logd/logd.cfg +++ b/configserver/src/main/resources/logd/logd.cfg @@ -1,3 +1,4 @@ +stateport 19089 logserver.use false loglevel.fatal.forward false loglevel.error.forward false diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/flags/FlagsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/flags/FlagsHandlerTest.java index 71caed77dbe..11958454a70 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/flags/FlagsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/flags/FlagsHandlerTest.java @@ -89,6 +89,10 @@ public class FlagsHandlerTest { verifySuccessfulRequest(Method.GET, "/data/", "", "{\"flags\":[{\"id\":\"id1\",\"url\":\"https://foo.com:4443/flags/v1/data/id1\"}]}"); + // Verify absent port => absent in response + assertThat(handleWithPort(Method.GET, -1, "/data", "", 200), + is("{\"flags\":[{\"id\":\"id1\",\"url\":\"https://foo.com/flags/v1/data/id1\"}]}")); + // PUT id2 verifySuccessfulRequest(Method.PUT, "/data/" + FLAG2.id(), "{\n" + @@ -175,7 +179,11 @@ public class FlagsHandlerTest { } private String handle(Method method, String pathSuffix, String requestBody, int expectedStatus) { - String uri = FLAGS_V1_URL + pathSuffix; + return handleWithPort(method, 4443, pathSuffix, requestBody, expectedStatus); + } + + private String handleWithPort(Method method, int port, String pathSuffix, String requestBody, int expectedStatus) { + String uri = "https://foo.com" + (port < 0 ? "" : ":" + port) + "/flags/v1" + pathSuffix; HttpRequest request = HttpRequest.createTestRequest(uri, method, makeInputStream(requestBody)); HttpResponse response = handler.handle(request); assertEquals(expectedStatus, response.getStatus()); diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java index bb4b57e8983..e84c8b340a4 100755 --- a/container-core/src/main/java/com/yahoo/container/Container.java +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -11,6 +11,7 @@ import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.osgi.Osgi; import com.yahoo.vespa.config.ConfigTransformer; +import com.yahoo.vespa.config.UrlDownloader; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -31,6 +32,7 @@ public class Container { private volatile ComponentRegistry<ServerProvider> serverProviderRegistry; private volatile ComponentRegistry<AbstractComponent> componentRegistry; private volatile FileAcquirer fileAcquirer; + private volatile UrlDownloader urlDownloader; private volatile BundleLoader bundleLoader; @@ -50,6 +52,8 @@ public class Container { public void shutdown() { if (fileAcquirer != null) fileAcquirer.shutdown(); + if (urlDownloader != null) + urlDownloader.shutdown(); } //Used to acquire files originating from the application package. @@ -147,4 +151,9 @@ public class Container { }); } + public void setupUrlDownloader() { + this.urlDownloader = new UrlDownloader(); + ConfigTransformer.setUrlDownloader(urlDownloader); + } + } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java index 557f331395b..0a97a4d5d2f 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java @@ -177,7 +177,7 @@ public class BundleLoader { private String installedBundlesMessage() { StringBuilder sb = new StringBuilder("Installed bundles: {" ); for (Bundle b : osgi.getBundles()) - sb.append("[" + b.getBundleId() + "]" + b.getSymbolicName() + ", "); + sb.append("[" + b.getBundleId() + "]" + b.getSymbolicName() + ":" + b.getVersion() + ", "); sb.setLength(sb.length() - 2); sb.append("}"); return sb.toString(); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index f4fae683877..1cb4e1d4555 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -191,6 +191,7 @@ public final class ConfiguredApplication implements Application { private static void hackToInitializeServer(QrConfig config) { try { Container.get().setupFileAcquirer(config.filedistributor()); + Container.get().setupUrlDownloader(); com.yahoo.container.Server.get().initialize(config); } catch (Exception e) { log.log(LogLevel.ERROR, "Caught exception when initializing server. Exiting.", e); diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 7e5fc873803..93a7320f8d1 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -2095,6 +2095,7 @@ "fields": [ "public static final com.yahoo.processing.request.CompoundName OFFSET", "public static final com.yahoo.processing.request.CompoundName HITS", + "public static final com.yahoo.processing.request.CompoundName QUERY_PROFILE", "public static final com.yahoo.processing.request.CompoundName SEARCH_CHAIN", "public static final com.yahoo.processing.request.CompoundName TRACE_LEVEL", "public static final com.yahoo.processing.request.CompoundName NO_CACHE", diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java index 131d4fcc9da..b87de5f019a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java @@ -49,6 +49,7 @@ public class WeightedSetItem extends SimpleTaggableItem { * Add weighted token. * If token is already in the set, the maximum weight is kept. * NOTE: The weight must be 1 or more; negative values (and zero) are not allowed. + * * @return weight of added token (might be old value, if kept) */ public Integer addToken(String token, int weight) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java index 652e1ca60b8..1157d2763e0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java @@ -15,7 +15,7 @@ import com.yahoo.compress.IntegerCompressor; * A set words with differing exactness scores to be used for literal boost * ranking. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class WordAlternativesItem extends TermItem { @@ -23,6 +23,7 @@ public class WordAlternativesItem extends TermItem { private int maxIndex; public static final class Alternative { + public final String word; public final double exactness; @@ -38,6 +39,7 @@ public class WordAlternativesItem extends TermItem { builder.append("Alternative [word=").append(word).append(", exactness=").append(exactness).append("]"); return builder.toString(); } + } public WordAlternativesItem(String indexName, boolean isFromQuery, Substring origin, Collection<Alternative> terms) { @@ -51,7 +53,7 @@ public class WordAlternativesItem extends TermItem { } private static ImmutableList<Alternative> uniqueAlternatives(Collection<Alternative> terms) { - List<Alternative> uniqueTerms = new ArrayList<Alternative>(terms.size()); + List<Alternative> uniqueTerms = new ArrayList<>(terms.size()); for (Alternative term : terms) { int i = Collections.binarySearch(uniqueTerms, term, (t0, t1) -> t0.word.compareTo(t1.word)); if (i >= 0) { @@ -104,7 +106,7 @@ public class WordAlternativesItem extends TermItem { @Override public void setValue(String value) { - throw new UnsupportedOperationException("semantics for setting to a string would be brittle, use setAlternatives()"); + throw new UnsupportedOperationException("Semantics for setting to a string would be brittle, use setAlternatives()"); } @Override @@ -180,4 +182,5 @@ public class WordAlternativesItem extends TermItem { newTerms.add(new Alternative(term, exactness)); setAlternatives(newTerms); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/textualrepresentation/Discloser.java b/container-search/src/main/java/com/yahoo/prelude/query/textualrepresentation/Discloser.java index 1d7372a2497..cbe1c0f8ab9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/textualrepresentation/Discloser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/textualrepresentation/Discloser.java @@ -9,9 +9,11 @@ import com.yahoo.prelude.query.Item; * @author Tony Vaagenes */ public interface Discloser { + void addProperty(String key, Object value); //A given item should either call setValue or addChild, not both. void setValue(Object value); void addChild(Item item); + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/textualrepresentation/TextualQueryRepresentation.java b/container-search/src/main/java/com/yahoo/prelude/query/textualrepresentation/TextualQueryRepresentation.java index 56f106a43f4..e299ccb5674 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/textualrepresentation/TextualQueryRepresentation.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/textualrepresentation/TextualQueryRepresentation.java @@ -8,11 +8,12 @@ import java.util.*; import java.util.regex.Pattern; /** - * Creates a detailed, QED inspired representation of a query tree. + * Creates a detailed representation of a query tree. * * @author Tony Vaagenes */ public class TextualQueryRepresentation { + private Map<Item, Integer> itemReferences = new IdentityHashMap<>(); private int nextItemReference = 0; @@ -207,4 +208,5 @@ public class TextualQueryRepresentation { public String toString() { return rootDiscloser.toString(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 8e5e14a3aac..bf0920a2aa5 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -107,14 +107,14 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private final int intValue; private final String stringValue; - Type(int intValue,String stringValue) { + Type(int intValue, String stringValue) { this.intValue = intValue; this.stringValue = stringValue; } /** Converts a type argument value into a query type */ public static Type getType(String typeString) { - for (Type type:Type.values()) + for (Type type : Type.values()) if (type.stringValue.equals(typeString)) return type; return ALL; @@ -187,6 +187,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static final CompoundName OFFSET = new CompoundName("offset"); public static final CompoundName HITS = new CompoundName("hits"); + public static final CompoundName QUERY_PROFILE = new CompoundName("queryProfile"); public static final CompoundName SEARCH_CHAIN = new CompoundName("searchChain"); public static final CompoundName TRACE_LEVEL = new CompoundName("traceLevel"); public static final CompoundName NO_CACHE = new CompoundName("noCache"); @@ -202,6 +203,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { argumentType.addField(new FieldDescription(OFFSET.toString(), "integer", "offset start")); argumentType.addField(new FieldDescription(HITS.toString(), "integer", "hits count")); // TODO: Should this be added to com.yahoo.search.query.properties.QueryProperties? If not, why not? + argumentType.addField(new FieldDescription(QUERY_PROFILE.toString(), "string")); argumentType.addField(new FieldDescription(SEARCH_CHAIN.toString(), "string")); argumentType.addField(new FieldDescription(TRACE_LEVEL.toString(), "integer", "tracelevel")); argumentType.addField(new FieldDescription(NO_CACHE.toString(), "boolean", "nocache")); @@ -329,8 +331,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { init(requestMap, queryProfile); } - - private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile) { startTime = System.currentTimeMillis(); if (queryProfile != null) { @@ -411,10 +411,10 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } } } else { - Object value=originalProperties.get(fullName,context); - if (value!=null) { + Object value = originalProperties.get(fullName,context); + if (value != null) { try { - properties().set(fullName,value,context); + properties().set(fullName, value, context); } catch (IllegalArgumentException e) { throw new QueryException("Invalid request parameter", e); } @@ -427,7 +427,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private void setPropertiesFromRequestMap(Map<String, String> requestMap, Properties properties) { for (Map.Entry<String, String> entry : requestMap.entrySet()) { try { - if (entry.getKey().equals("queryProfile")) continue; properties.set(entry.getKey(), entry.getValue(), requestMap); } catch (IllegalArgumentException e) { @@ -446,12 +445,12 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ private void traceProperties() { if (traceLevel == 0) return; - CompiledQueryProfile profile=null; - QueryProfileProperties profileProperties=properties().getInstance(QueryProfileProperties.class); - if (profileProperties!=null) - profile=profileProperties.getQueryProfile(); + CompiledQueryProfile profile = null; + QueryProfileProperties profileProperties = properties().getInstance(QueryProfileProperties.class); + if (profileProperties != null) + profile = profileProperties.getQueryProfile(); - if (profile==null) + if (profile == null) trace("No query profile is used", false, 1); else trace("Using " + profile.toString(), false, 1); @@ -466,7 +465,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { b.append(requestProperty.getKey()); b.append("="); - b.append(String.valueOf(resolvedValue)); // (may be null) + b.append(resolvedValue); // (may be null) b.append(" ("); if (profile != null && ! profile.isOverridable(new CompoundName(requestProperty.getKey()), requestProperties())) @@ -476,8 +475,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { b.append(")\n"); mentioned.add(requestProperty.getKey()); } - if (profile!=null) { - appendQueryProfileProperties(profile,mentioned,b); + if (profile != null) { + appendQueryProfileProperties(profile, mentioned, b); } trace(b.toString(),false,4); } @@ -487,7 +486,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } private void appendQueryProfileProperties(CompiledQueryProfile profile,Set<String> mentioned,StringBuilder b) { - for (Map.Entry<String,Object> property : profile.listValues("",requestProperties()).entrySet()) { + for (Map.Entry<String,Object> property : profile.listValues("", requestProperties()).entrySet()) { if ( ! mentioned.contains(property.getKey())) b.append(property.getKey() + "=" + property.getValue() + " (value from query profile)<br/>\n"); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index fc1da407ae7..6f965944bdf 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -579,7 +579,8 @@ public class SearchHandler extends LoggingRequestHandler { byte[] byteArray = IOUtils.readBytes(request.getData(), 1 << 20); inspector = SlimeUtils.jsonToSlime(byteArray).get(); if (inspector.field("error_message").valid()){ - throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8)); + throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + ", at: " + + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8)); } } catch (IOException e) { @@ -631,7 +632,7 @@ public class SearchHandler extends LoggingRequestHandler { map.put(qualifiedKey, value.asString()); break; case OBJECT: - if (qualifiedKey.equals("select.where") || qualifiedKey.equals("select.grouping")){ + if (qualifiedKey.equals("select.where") || qualifiedKey.equals("select.grouping")) { map.put(qualifiedKey, value.toString()); break; } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index 4baa651fa01..fd52618ad85 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -7,7 +7,6 @@ import com.yahoo.language.LocaleFactory; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.TaggableItem; -import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.query.parser.Parsable; @@ -18,7 +17,13 @@ import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.searchchain.Execution; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; import static com.yahoo.text.Lowercase.toLowerCase; @@ -169,9 +174,9 @@ public class Model implements Cloneable { public void setLanguage(Language language) { this.language = language; } /** - * <p>Explicitly sets the language to be used during parsing. The argument is first normalized by replacing + * Explicitly sets the language to be used during parsing. The argument is first normalized by replacing * underscores with hyphens (to support locale strings being used as RFC 5646 language tags), and then forwarded to - * {@link #setLocale(String)} so that the Locale information of the tag is preserved.</p> + * {@link #setLocale(String)} so that the Locale information of the tag is preserved. * * @param language The language string to parse. * @see #getLanguage() @@ -182,9 +187,9 @@ public class Model implements Cloneable { } /** - * <p>Returns the explicitly set parsing locale of this query model, or null if none.</p> + * Returns the explicitly set parsing locale of this query model, or null if none. * - * @return The locale of this. + * @return the locale of this * @see #setLocale(Locale) */ public Locale getLocale() { @@ -195,7 +200,7 @@ public class Model implements Cloneable { * <p>Explicitly sets the locale to be used during parsing. This method also calls {@link #setLanguage(Language)} * with the corresponding {@link Language} instance.</p> * - * @param locale The locale to set. + * @param locale the locale to set * @see #getLocale() * @see #setLanguage(Language) */ @@ -205,10 +210,10 @@ public class Model implements Cloneable { } /** - * <p>Explicitly sets the locale to be used during parsing. This creates a Locale instance from the given language - * tag, and passes that to {@link #setLocale(Locale)}.</p> + * Explicitly sets the locale to be used during parsing. This creates a Locale instance from the given language + * tag, and passes that to {@link #setLocale(Locale)}. * - * @param languageTag The language tag to parse. + * @param languageTag the language tag to parse * @see #setLocale(Locale) */ public void setLocale(String languageTag) { diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index bbc152c6391..f2a534a014e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -88,8 +88,7 @@ public class Select implements Cloneable { } /** Returns the where clause string previously assigned, or an empty string if none */ - public String getWhereString(){ return where; } - + public String getWhereString() { return where; } /** * Sets the grouping operation of the query. @@ -120,7 +119,6 @@ public class Select implements Cloneable { */ public List<GroupingRequest> getGrouping() { return groupingRequests; } - @Override public String toString() { return "where: [" + where + "], grouping: [" + grouping+ "]"; diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 13ebacb62ef..e4e44985b53 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -68,8 +68,6 @@ import static com.yahoo.slime.Type.STRING; * * @author henrhoi */ - - public class SelectParser implements Parser { Parsable query; @@ -77,13 +75,9 @@ public class SelectParser implements Parser { private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap(); private final List<ConnectedItem> connectedItems = new ArrayList<>(); private final Normalizer normalizer; - private final ParserEnvironment environment; private IndexFacts.Session indexFactsSession; - - - /** YQL parameters and functions */ - + // YQL parameters and functions private static final String DESCENDING_HITS_ORDER = "descending"; private static final String ASCENDING_HITS_ORDER = "ascending"; private static final Integer DEFAULT_TARGET_NUM_HITS = 10; @@ -139,18 +133,11 @@ public class SelectParser implements Parser { private static final String CALL = "call"; private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND); - /**************************************/ - - - public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); normalizer = environment.getLinguistics().getNormalizer(); - - this.environment = environment; } - @Override public QueryTree parse(Parsable query) { indexFactsSession = indexFacts.newSession(query.getSources(), query.getRestrict()); @@ -161,8 +148,6 @@ public class SelectParser implements Parser { return buildTree(); } - - private QueryTree buildTree() { Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get(); if (inspector.field("error_message").valid()){ @@ -176,14 +161,12 @@ public class SelectParser implements Parser { return newTree; } - private Item walkJson(Inspector inspector){ - final Item[] item = {null}; + Item[] item = {null}; inspector.traverse((ObjectTraverser) (key, value) -> { String type = (FUNCTION_CALLS.contains(key)) ? CALL : key; switch (type) { - case AND: item[0] = buildAnd(key, value); break; @@ -215,7 +198,6 @@ public class SelectParser implements Parser { return item[0]; } - public List<VespaGroupingStep> getGroupingSteps(String grouping){ List<VespaGroupingStep> groupingSteps = new ArrayList<>(); List<String> groupingOperations = getOperations(grouping); @@ -242,10 +224,8 @@ public class SelectParser implements Parser { }); return operations; - } - @NonNull private Item buildFunctionCall(String key, Inspector value) { switch (key) { @@ -266,7 +246,6 @@ public class SelectParser implements Parser { } } - private void addItemsFromInspector(CompositeItem item, Inspector inspector){ if (inspector.type() == ARRAY){ inspector.traverse((ArrayTraverser) (index, new_value) -> { @@ -283,7 +262,6 @@ public class SelectParser implements Parser { } } - private Inspector getChildren(Inspector inspector){ if (inspector.type() == ARRAY){ return inspector; @@ -299,25 +277,23 @@ public class SelectParser implements Parser { return null; } - private HashMap<Integer, Inspector> getChildrenMap(Inspector inspector){ HashMap<Integer, Inspector> children = new HashMap<>(); - if (inspector.type() == ARRAY){ - inspector.traverse((ArrayTraverser) (index, new_value) -> { + if (inspector.type() == ARRAY){ + inspector.traverse((ArrayTraverser) (index, new_value) -> { + children.put(index, new_value); + }); + + } else if (inspector.type() == OBJECT){ + if (inspector.field("children").valid()){ + inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { children.put(index, new_value); }); - - } else if (inspector.type() == OBJECT){ - if (inspector.field("children").valid()){ - inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { - children.put(index, new_value); - }); - } } + } return children; } - private Inspector getAnnotations(Inspector inspector){ if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ return inspector.field("attributes"); @@ -325,7 +301,6 @@ public class SelectParser implements Parser { return null; } - private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation){ HashMap<String, Inspector> attributes = new HashMap<>(); if (annotation.type() == OBJECT){ @@ -336,7 +311,6 @@ public class SelectParser implements Parser { return attributes; } - private HashMap<String, Inspector> getAnnotationMap(Inspector inspector){ HashMap<String, Inspector> attributes = new HashMap<>(); if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ @@ -347,12 +321,10 @@ public class SelectParser implements Parser { return attributes; } - private <T> T getAnnotation(String annotationName, HashMap<String, Inspector> annotations, Class<T> expectedClass, T defaultValue) { return (annotations.get(annotationName) == null) ? defaultValue : expectedClass.cast(annotations.get(annotationName).asString()); } - private Boolean getBoolAnnotation(String annotationName, HashMap<String, Inspector> annotations, Boolean defaultValue) { if (annotations != null){ Inspector annotation = annotations.getOrDefault(annotationName, null); @@ -363,7 +335,6 @@ public class SelectParser implements Parser { return defaultValue; } - private Integer getIntegerAnnotation(String annotationName, HashMap<String, Inspector> annotations, Integer defaultValue) { if (annotations != null){ Inspector annotation = annotations.getOrDefault(annotationName, null); @@ -374,7 +345,6 @@ public class SelectParser implements Parser { return defaultValue; } - private Double getDoubleAnnotation(String annotationName, HashMap<String, Inspector> annotations, Double defaultValue) { if (annotations != null){ Inspector annotation = annotations.getOrDefault(annotationName, null); @@ -385,12 +355,10 @@ public class SelectParser implements Parser { return defaultValue; } - private Inspector getAnnotationAsInspectorOrNull(String annotationName, HashMap<String, Inspector> annotations) { return annotations.get(annotationName); } - @NonNull private CompositeItem buildAnd(String key, Inspector value) { AndItem andItem = new AndItem(); @@ -399,7 +367,6 @@ public class SelectParser implements Parser { return andItem; } - @NonNull private CompositeItem buildNotAnd(String key, Inspector value) { NotItem notItem = new NotItem(); @@ -408,7 +375,6 @@ public class SelectParser implements Parser { return notItem; } - @NonNull private CompositeItem buildOr(String key, Inspector value) { OrItem orItem = new OrItem(); @@ -416,7 +382,6 @@ public class SelectParser implements Parser { return orItem; } - @NonNull private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); @@ -437,7 +402,6 @@ public class SelectParser implements Parser { return weakAnd; } - @NonNull private <T extends TaggableItem> T leafStyleSettings(Inspector annotations, @NonNull T out) { { @@ -521,9 +485,8 @@ public class SelectParser implements Parser { return out; } - private Integer getCappedRangeSearchParameter(Inspector annotations) { - final Integer[] hitLimit = {null}; + Integer[] hitLimit = {null}; annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { if (HIT_LIMIT.equals(annotation_name)) { if (annotation_value != null) { @@ -531,8 +494,8 @@ public class SelectParser implements Parser { } } }); - final Boolean[] ascending = {null}; - final Boolean[] descending = {null}; + Boolean[] ascending = {null}; + Boolean[] descending = {null}; if (hitLimit[0] != null) { annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { @@ -554,13 +517,12 @@ public class SelectParser implements Parser { return hitLimit[0]; } - @NonNull private Item buildRange(String key, Inspector value) { HashMap<Integer, Inspector> children = getChildrenMap(value); Inspector annotations = getAnnotations(value); - final boolean[] equals = {false}; + boolean[] equals = {false}; String field; Inspector boundInspector; @@ -572,8 +534,8 @@ public class SelectParser implements Parser { boundInspector = children.get(0); } - final Number[] bounds = {null, null}; - final String[] operators = {null, null}; + Number[] bounds = {null, null}; + String[] operators = {null, null}; boundInspector.traverse((ObjectTraverser) (operator, bound) -> { if (bound.type() == STRING) { throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."); @@ -625,26 +587,22 @@ public class SelectParser implements Parser { } - @NonNull private IntItem buildLessThanOrEquals(String field, String bound) { return new IntItem("[;" + bound + "]", field); } - @NonNull private IntItem buildGreaterThan(String field, String bound) { return new IntItem(">" + bound, field); } - @NonNull private IntItem buildLessThan(String field, String bound) { return new IntItem("<" + bound, field); } - @NonNull private IntItem instantiateRangeItem(Number lowerBound, Number upperBound, String field, boolean bounds_left_open, boolean bounds_right_open) { Preconditions.checkArgument(lowerBound != null && upperBound != null && field != null, @@ -675,7 +633,6 @@ public class SelectParser implements Parser { return buildRange(key, value); } - @NonNull private Item buildWand(String key, Inspector value) { HashMap<String, Inspector> annotations = getAnnotationMap(value); @@ -699,7 +656,6 @@ public class SelectParser implements Parser { return fillWeightedSet(value, children, out); } - @NonNull private WeightedSetItem fillWeightedSet(Inspector value, HashMap<Integer, Inspector> children, @NonNull WeightedSetItem out) { addItems(children, out); @@ -721,7 +677,6 @@ public class SelectParser implements Parser { } } - private static void addStringItems(HashMap<Integer, Inspector> children, WeightedSetItem out) { //{"a":1, "b":2} children.get(1).traverse((ObjectTraverser) (key, value) -> { @@ -732,9 +687,7 @@ public class SelectParser implements Parser { }); } - private static void addLongItems(HashMap<Integer, Inspector> children, WeightedSetItem out) { - //[[11,1], [37,2]] children.get(1).traverse((ArrayTraverser) (index, pair) -> { List<Integer> pairValues = new ArrayList<>(); pair.traverse((ArrayTraverser) (pairIndex, pairValue) -> { @@ -746,7 +699,6 @@ public class SelectParser implements Parser { }); } - @NonNull private Item buildRegExpSearch(String key, Inspector value) { assertHasOperator(key, MATCHES); @@ -757,7 +709,6 @@ public class SelectParser implements Parser { return leafStyleSettings(getAnnotations(value), regExp); } - @NonNull private Item buildWeightedSet(String key, Inspector value) { HashMap<Integer, Inspector> children = getChildrenMap(value); @@ -766,7 +717,6 @@ public class SelectParser implements Parser { return fillWeightedSet(value, children, new WeightedSetItem(field)); } - @NonNull private Item buildDotProduct(String key, Inspector value) { HashMap<Integer, Inspector> children = getChildrenMap(value); @@ -775,7 +725,6 @@ public class SelectParser implements Parser { return fillWeightedSet(value, children, new DotProductItem(field)); } - @NonNull private Item buildPredicate(String key, Inspector value) { HashMap<Integer, Inspector> children = getChildrenMap(value); @@ -805,7 +754,6 @@ public class SelectParser implements Parser { return leafStyleSettings(getAnnotations(value), item); } - @NonNull private CompositeItem buildRank(String key, Inspector value) { RankItem rankItem = new RankItem(); @@ -813,7 +761,6 @@ public class SelectParser implements Parser { return rankItem; } - @NonNull private Item buildTermSearch(String key, Inspector value) { HashMap<Integer, Inspector> children = getChildrenMap(value); @@ -822,7 +769,6 @@ public class SelectParser implements Parser { return instantiateLeafItem(field, key, value); } - private String getInspectorKey(Inspector inspector){ String[] actualKey = {""}; if (inspector.type() == OBJECT){ @@ -834,7 +780,6 @@ public class SelectParser implements Parser { return actualKey[0]; } - @NonNull private Item instantiateLeafItem(String field, String key, Inspector value) { List<Inspector> possibleLeafFunction = valueListFromInspector(value); @@ -848,7 +793,6 @@ public class SelectParser implements Parser { } } - @NonNull private Item instantiateCompositeLeaf(String field, String key, Inspector value) { switch (key) { @@ -869,14 +813,12 @@ public class SelectParser implements Parser { } } - @NonNull private Item instantiateWordItem(String field, String key, Inspector value) { String wordData = getChildrenMap(value).get(1).asString(); return instantiateWordItem(field, wordData, key, value, false, decideParsingLanguage(value, wordData)); } - @NonNull private Item instantiateWordItem(String field, String rawWord, String key, Inspector value, boolean exactMatch, Language language) { String wordData = rawWord; @@ -919,7 +861,6 @@ public class SelectParser implements Parser { return (Item) leafStyleSettings(getAnnotations(value), wordItem); } - private Language decideParsingLanguage(Inspector value, String wordData) { String languageTag = getAnnotation(USER_INPUT_LANGUAGE, getAnnotationMap(value), String.class, null); @@ -932,13 +873,11 @@ public class SelectParser implements Parser { return Language.ENGLISH; } - private void prepareWord(String field, Inspector value, WordItem wordItem) { wordItem.setIndexName(field); wordStyleSettings(value, wordItem); } - private void wordStyleSettings(Inspector value, WordItem out) { HashMap<String, Inspector> annotations = getAnnotationMap(value); @@ -947,7 +886,7 @@ public class SelectParser implements Parser { out.setOrigin(origin); } if (annotations != null){ - Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null)); + Boolean usePositionData = getBoolAnnotation(USE_POSITION_DATA, annotations, null); if (usePositionData != null) { out.setPositionData(usePositionData); } @@ -975,16 +914,15 @@ public class SelectParser implements Parser { } } - private Substring getOrigin(Inspector annotations) { if (annotations != null) { Inspector origin = getAnnotationAsInspectorOrNull(ORIGIN, getAnnotationMapFromAnnotationInspector(annotations)); if (origin == null) { return null; } - final String[] original = {null}; - final Integer[] offset = {null}; - final Integer[] length = {null}; + String[] original = {null}; + Integer[] offset = {null}; + Integer[] length = {null}; origin.traverse((ObjectTraverser) (key, value) -> { switch (key) { @@ -1020,25 +958,23 @@ public class SelectParser implements Parser { return sameElement; } - @NonNull private Item instantiatePhraseItem(String field, String key, Inspector value) { assertHasOperator(key, PHRASE); - HashMap<String, Inspector> annotations = getAnnotationMap(value); PhraseItem phrase = new PhraseItem(); phrase.setIndexName(field); HashMap<Integer, Inspector> children = getChildrenMap(value); - for (Inspector word : children.values()) - if (word.type() == STRING) phrase.addItem(new WordItem(word.asString())); - else if (word.type() == OBJECT && word.field(PHRASE).valid()) { - phrase.addItem(instantiatePhraseItem(field, key, getChildren(word))); - } + for (Inspector word : children.values()) { + if (word.type() == STRING) + phrase.addItem(new WordItem(word.asString())); + else if (word.type() == OBJECT && word.field(PHRASE).valid()) + phrase.addItem(instantiatePhraseItem(field, key, getChildren(word))); + } return leafStyleSettings(getAnnotations(value), phrase); } - @NonNull private Item instantiateNearItem(String field, String key, Inspector value) { assertHasOperator(key, NEAR); @@ -1060,7 +996,6 @@ public class SelectParser implements Parser { return near; } - @NonNull private Item instantiateONearItem(String field, String key, Inspector value) { assertHasOperator(key, ONEAR); @@ -1105,7 +1040,6 @@ public class SelectParser implements Parser { return leafStyleSettings(getAnnotations(value), equiv); } - private Item instantiateWordAlternativesItem(String field, String key, Inspector value) { HashMap<Integer, Inspector> children = getChildrenMap(value); Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size()); @@ -1119,7 +1053,6 @@ public class SelectParser implements Parser { return leafStyleSettings(getAnnotations(value), new WordAlternativesItem(field, Boolean.TRUE, null, terms)); } - // Not in use yet @NonNull private String getIndex(String field) { @@ -1128,12 +1061,10 @@ public class SelectParser implements Parser { return field; } - private static void assertHasOperator(String key, String expectedKey) { Preconditions.checkArgument(key.equals(expectedKey), "Expected operator %s, got %s.", expectedKey, key); } - private static IllegalArgumentException newUnexpectedArgumentException(Object actual, Object... expected) { StringBuilder out = new StringBuilder("Expected "); for (int i = 0, len = expected.length; i < len; ++i) { @@ -1148,26 +1079,22 @@ public class SelectParser implements Parser { return new IllegalArgumentException(out.toString()); } - private List<Inspector> valueListFromInspector(Inspector inspector){ List<Inspector> inspectorList = new ArrayList<>(); inspector.traverse((ArrayTraverser) (key, value) -> inspectorList.add(value)); return inspectorList; } - private void connectItems() { for (ConnectedItem entry : connectedItems) { TaggableItem to = identifiedItems.get(entry.toId); Preconditions.checkNotNull(to, "Item '%s' was specified to connect to item with ID %s, which does not " - + "exist in the query.", entry.fromItem, - entry.toId); + + "exist in the query.", entry.fromItem, entry.toId); entry.fromItem.setConnectivity((Item) to, entry.weight); } } - private static final class ConnectedItem { final double weight; @@ -1181,5 +1108,4 @@ public class SelectParser implements Parser { } } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java index 36b38ad8d03..3252f0f4662 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java @@ -27,23 +27,23 @@ public class SubstituteString { * Returns a new SubstituteString if the given string contains substitutions, null otherwise. */ public static SubstituteString create(String value) { - int lastEnd=0; - int start=value.indexOf("%{"); - if (start<0) return null; // Shortcut - List<Component> components=new ArrayList<>(); - while (start>=0) { - int end=value.indexOf("}",start+2); - if (end<0) + int lastEnd = 0; + int start = value.indexOf("%{"); + if (start < 0) return null; // Shortcut + List<Component> components = new ArrayList<>(); + while (start >= 0) { + int end = value.indexOf("}", start + 2); + if (end < 0) throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'"); - String propertyName=value.substring(start+2,end); - if (propertyName.indexOf("%{")>=0) + String propertyName = value.substring(start+2,end); + if (propertyName.indexOf("%{") >= 0) throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'"); - components.add(new StringComponent(value.substring(lastEnd,start))); + components.add(new StringComponent(value.substring(lastEnd, start))); components.add(new PropertyComponent(propertyName)); - lastEnd=end+1; - start=value.indexOf("%{",lastEnd); + lastEnd = end+1; + start = value.indexOf("%{", lastEnd); } - components.add(new StringComponent(value.substring(lastEnd,value.length()))); + components.add(new StringComponent(value.substring(lastEnd))); return new SubstituteString(components, value); } @@ -83,7 +83,7 @@ public class SubstituteString { private abstract static class Component { - protected abstract String getValue(Map<String,String> context,Properties substitution); + protected abstract String getValue(Map<String, String> context, Properties substitution); } @@ -92,11 +92,11 @@ public class SubstituteString { private final String value; public StringComponent(String value) { - this.value=value; + this.value = value; } @Override - public String getValue(Map<String,String> context,Properties substitution) { + public String getValue(Map<String, String> context, Properties substitution) { return value; } @@ -112,13 +112,13 @@ public class SubstituteString { private final String propertyName; public PropertyComponent(String propertyName) { - this.propertyName=propertyName; + this.propertyName = propertyName; } @Override - public String getValue(Map<String,String> context,Properties substitution) { - Object value=substitution.get(propertyName,context,substitution); - if (value==null) return ""; + public String getValue(Map<String,String> context, Properties substitution) { + Object value = substitution.get(propertyName, context, substitution); + if (value == null) return ""; return String.valueOf(value); } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 55855624691..60427aeb0af 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -3,9 +3,12 @@ package com.yahoo.search.query.properties; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; -import com.yahoo.search.grouping.GroupingRequest; -import com.yahoo.search.grouping.vespa.GroupingExecutor; -import com.yahoo.search.query.*; + +import com.yahoo.search.query.Model; +import com.yahoo.search.query.Presentation; +import com.yahoo.search.query.Properties; +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.Select; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -15,11 +18,8 @@ import com.yahoo.search.query.ranking.Matching; import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.tensor.Tensor; -import java.util.List; import java.util.Map; - - /** * Maps between the query model and text properties. * This can be done simpler by using reflection but the performance penalty was not worth it, diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/PhraseMatchTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/PhraseMatchTestCase.java index 41f67ed16fc..5cee88de849 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/PhraseMatchTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/PhraseMatchTestCase.java @@ -22,4 +22,9 @@ public class PhraseMatchTestCase extends RuleBaseAbstractTestCase { assertSemantics("AND retailer:digital retailer:camera","keyword:digital keyword:camera"); } + @Test + public void testMatchingPhrase() { + assertSemantics("OR (AND iphone 7) i7", "iphone 7"); + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/phrasematch.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/phrasematch.sr index f985c693284..70351ba8ba1 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/phrasematch.sr +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/phrasematch.sr @@ -3,3 +3,5 @@ [ret] :- keyword:[B]; retailer:"[...]" -> retailer:[...]; + +iphone 7 +> ?i7;
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index e0fef7fc3b4..fa398efd293 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -13,6 +13,7 @@ import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; import com.yahoo.slime.Inspector; import com.yahoo.vespa.config.SlimeUtils; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.After; import org.junit.Before; @@ -30,7 +31,11 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; - +/** + * Tests submitting the query as JSON. + * + * @author henrhoi + */ public class JSONSearchHandlerTestCase { private static final String testDir = "src/test/java/com/yahoo/search/handler/test/config"; @@ -154,7 +159,7 @@ public class JSONSearchHandlerTestCase { } } - private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) throws Exception{ + private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) throws Exception { JSONObject json = new JSONObject(); json.put("query", "status_code:0"); json.put("hits", 20); @@ -167,9 +172,6 @@ public class JSONSearchHandlerTestCase { assertThat(response, containsString("\"code\":" + com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code)); } - - - @Test public void testNormalResultJsonAliasRendering() throws Exception { JSONObject json = new JSONObject(); @@ -178,8 +180,6 @@ public class JSONSearchHandlerTestCase { assertJsonResult(json, driver); } - - @Test public void testNullQuery() throws Exception { JSONObject json = new JSONObject(); @@ -194,8 +194,6 @@ public class JSONSearchHandlerTestCase { "</result>\n", driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE).readAll()); } - - @Test public void testWebServiceStatus() throws Exception { JSONObject json = new JSONObject(); @@ -230,7 +228,6 @@ public class JSONSearchHandlerTestCase { assertXmlResult(json, driver); } - @Test public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { JSONObject json = new JSONObject(); @@ -263,7 +260,6 @@ public class JSONSearchHandlerTestCase { assertPageResult(json, driver); } - private static final String xmlResult = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<result total-hit-count=\"0\">\n" + @@ -273,18 +269,17 @@ public class JSONSearchHandlerTestCase { " </hit>\n" + "</result>\n"; - private void assertXmlResult(JSONObject json, RequestHandlerTestDriver driver) throws Exception { + private void assertXmlResult(JSONObject json, RequestHandlerTestDriver driver) { assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), xmlResult); } - private static final String jsonResult = "{\"root\":{" + "\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0}," + "\"children\":[" + "{\"id\":\"testHit\",\"relevance\":1.0,\"fields\":{\"uri\":\"testHit\"}}" + "]}}"; - private void assertJsonResult(JSONObject json, RequestHandlerTestDriver driver) throws Exception { + private void assertJsonResult(JSONObject json, RequestHandlerTestDriver driver) { assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), jsonResult); } @@ -300,7 +295,7 @@ public class JSONSearchHandlerTestCase { "\n" + "</result>\n"; - private void assertTiledResult(JSONObject json, RequestHandlerTestDriver driver) throws Exception { + private void assertTiledResult(JSONObject json, RequestHandlerTestDriver driver) { assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), tiledResult); } @@ -317,7 +312,7 @@ public class JSONSearchHandlerTestCase { "\n" + "</page>\n"; - private void assertPageResult(JSONObject json, RequestHandlerTestDriver driver) throws Exception { + private void assertPageResult(JSONObject json, RequestHandlerTestDriver driver) { assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), pageResult); } @@ -338,9 +333,8 @@ public class JSONSearchHandlerTestCase { return new RequestHandlerTestDriver(newSearchHandler); } - @Test - public void testSelectParameter() throws Exception { + public void testSelectParameters() throws Exception { JSONObject json = new JSONObject(); JSONObject select = new JSONObject(); @@ -356,8 +350,6 @@ public class JSONSearchHandlerTestCase { json.put("select", select); - - // Create mapping Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes("utf-8")).get(); Map<String, String> map = new HashMap<>(); searchHandler.createRequestMapping(inspector, map, ""); @@ -369,7 +361,34 @@ public class JSONSearchHandlerTestCase { assertEquals(grouping.toString(), processedGrouping.toString()); } + @Test + public void testJsonQueryWithSelectWhere() throws Exception { + JSONObject root = new JSONObject(); + JSONObject select = new JSONObject(); + JSONObject where = new JSONObject(); + JSONArray term = new JSONArray(); + term.put("default"); + term.put("bad"); + where.put("contains", term); + select.put("where", where); + root.put("select", select); + + // Run query + String result = driver.sendRequest(uri + "searchChain=echoingQuery", com.yahoo.jdisc.http.HttpRequest.Method.POST, root.toString(), JSON_CONTENT_TYPE).readAll(); + assertEquals("{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where default contains \\\"bad\\\";\"}}]}}", + result); + } + @Test + public void testJsonQueryWithYQL() throws Exception { + JSONObject root = new JSONObject(); + root.put("yql", "select * from sources * where default contains 'bad';"); + + // Run query + String result = driver.sendRequest(uri + "searchChain=echoingQuery", com.yahoo.jdisc.http.HttpRequest.Method.POST, root.toString(), JSON_CONTENT_TYPE).readAll(); + assertEquals("{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where default contains \\\"bad\\\";\"}}]}}", + result); + } @Test public void testRequestMapping() throws Exception { @@ -468,8 +487,6 @@ public class JSONSearchHandlerTestCase { json.put("nocachewrite", false); json.put("hitcountestimate", true); - - // Create mapping Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes("utf-8")).get(); Map<String, String> map = new HashMap<>(); @@ -484,9 +501,7 @@ public class JSONSearchHandlerTestCase { "&queryProfile=foo&presentation.bolding=true&model.encoding=json&model.queryString=abc&streaming.selection=none&trace.timestamps=false&collapse.size=2&streaming.priority=10&ranking.matchPhase.diversity.attribute=title" + "&ranking.matchPhase.attribute=title&hits=10&streaming.userid=123&pos.bb=1237123W%3B123218N&model.restrict=_doc%2Cjson%2Cxml&ranking.freshness=0.05&user=123"; - - - final HttpRequest request = HttpRequest.createTestRequest(url, GET); + HttpRequest request = HttpRequest.createTestRequest(url, GET); // Get mapping Map<String, String> propertyMap = request.propertyMap(); diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java index 6dcb34ec3e9..5ef13eba2ed 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java @@ -406,6 +406,19 @@ public class SearchHandlerTestCase { } /** Referenced from config */ + public static class EchoingQuerySearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + Hit hit = new Hit("Query"); + hit.setField("query", query.yqlRepresentation()); + result.hits().add(hit); + return result; + } + } + + /** Referenced from config */ public static class ForwardingHandler extends ThreadedHttpRequestHandler { private final SearchHandler searchHandler; diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg index 0336e06f54b..9a16c6ed1e7 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg @@ -1,4 +1,4 @@ -chains[3] +chains[4] chains[0].id default chains[0].components[1] chains[0].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher @@ -8,7 +8,13 @@ chains[1].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$Clas chains[2].id exceptionInPlugin chains[2].components[1] chains[2].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$ExceptionInPluginSearcher -components[3] +chains[3].id echoingQuery +chains[3].components[2] +chains[3].components[0] com.yahoo.search.yql.MinimalQueryInserter +chains[3].components[1] com.yahoo.search.handler.test.SearchHandlerTestCase$EchoingQuerySearcher +components[5] components[0].id com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher components[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$ClassLoadingErrorSearcher components[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$ExceptionInPluginSearcher +components[3].id com.yahoo.search.handler.test.SearchHandlerTestCase$EchoingQuerySearcher +components[4].id com.yahoo.search.yql.MinimalQueryInserter diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java index 67fb5da10a0..67d22fba4a3 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java @@ -80,7 +80,7 @@ public class QueryProfileIntegrationTestCase { System.setProperty("config.id", configId); Container container = new Container(); HandlersConfigurerTestWrapper configurer = new HandlersConfigurerTestWrapper(container, configId); - SearchHandler searchHandler = (SearchHandler) configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + SearchHandler searchHandler = (SearchHandler)configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); // Should get "default" query profile containing the "test" search chain containing the "test" searcher HttpRequest request = HttpRequest.createTestRequest("search", Method.GET); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java index 7ff120ddc70..9a0063e7f07 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java @@ -19,7 +19,7 @@ public class QueryProfileSubstitutionTestCase { @Test public void testSingleSubstitution() { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message","Hello %{world}!", null); p.set("world", "world", null); assertEquals("Hello world!",p.compile(null).get("message")); @@ -27,7 +27,7 @@ public class QueryProfileSubstitutionTestCase { QueryProfile p2=new QueryProfile("test2"); p2.addInherited(p); p2.set("world", "universe", null); - assertEquals("Hello universe!",p2.compile(null).get("message")); + assertEquals("Hello universe!", p2.compile(null).get("message")); } @Test @@ -39,16 +39,16 @@ public class QueryProfileSubstitutionTestCase { p.set("exclamation","?", null); assertEquals("Hola local group?",p.compile(null).get("message")); - QueryProfile p2=new QueryProfile("test2"); + QueryProfile p2 = new QueryProfile("test2"); p2.addInherited(p); p2.set("entity","milky way", null); - assertEquals("Hola milky way?",p2.compile(null).get("message")); + assertEquals("Hola milky way?", p2.compile(null).get("message")); } @Test public void testUnclosedSubstitution1() { try { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message1","%{greeting} %{entity}%{exclamation", null); fail("Should have produced an exception"); } @@ -61,7 +61,7 @@ public class QueryProfileSubstitutionTestCase { @Test public void testUnclosedSubstitution2() { try { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message1","%{greeting} %{entity%{exclamation}", null); fail("Should have produced an exception"); } @@ -73,26 +73,26 @@ public class QueryProfileSubstitutionTestCase { @Test public void testNullSubstitution() { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message","%{greeting} %{entity}%{exclamation}", null); p.set("greeting","Hola", null); assertEquals("Hola ", p.compile(null).get("message")); - QueryProfile p2=new QueryProfile("test2"); + QueryProfile p2 = new QueryProfile("test2"); p2.addInherited(p); p2.set("greeting","Hola", null); p2.set("exclamation", "?", null); - assertEquals("Hola ?",p2.compile(null).get("message")); + assertEquals("Hola ?", p2.compile(null).get("message")); } @Test public void testNoOverridingOfPropertiesSetAtRuntime() { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message","Hello %{world}!", null); p.set("world","world", null); p.freeze(); - Properties runtime=new QueryProfileProperties(p.compile(null)); + Properties runtime = new QueryProfileProperties(p.compile(null)); runtime.set("runtimeMessage","Hello %{world}!"); assertEquals("Hello world!", runtime.get("message")); assertEquals("Hello %{world}!",runtime.get("runtimeMessage")); @@ -100,18 +100,18 @@ public class QueryProfileSubstitutionTestCase { @Test public void testButPropertiesSetAtRuntimeAreUsedInSubstitutions() { - QueryProfile p=new QueryProfile("test"); - p.set("message","Hello %{world}!", null); - p.set("world","world", null); + QueryProfile p = new QueryProfile("test"); + p.set("message", "Hello %{world}!", null); + p.set("world", "world", null); - Properties runtime=new QueryProfileProperties(p.compile(null)); - runtime.set("world","Earth"); - assertEquals("Hello Earth!",runtime.get("message")); + Properties runtime = new QueryProfileProperties(p.compile(null)); + runtime.set("world", "Earth"); + assertEquals("Hello Earth!", runtime.get("message")); } @Test public void testInspection() { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message", "%{greeting} %{entity}%{exclamation}", null); assertEquals("message","%{greeting} %{entity}%{exclamation}", p.declaredContent().entrySet().iterator().next().getValue().toString()); @@ -119,7 +119,7 @@ public class QueryProfileSubstitutionTestCase { @Test public void testVariants() { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message","Hello %{world}!", null); p.set("world","world", null); p.setDimensions(new String[] {"x"}); @@ -134,7 +134,7 @@ public class QueryProfileSubstitutionTestCase { @Test public void testRecursion() { - QueryProfile p=new QueryProfile("test"); + QueryProfile p = new QueryProfile("test"); p.set("message","Hello %{world}!", null); p.set("world","sol planet number %{number}", null); p.set("number",3, null); diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index e60d84db3d0..ed80c0bf256 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -300,6 +300,14 @@ public class QueryTestCase { } @Test + public void testQueryProfileSubstitution() { + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("myField", "Profile: %{queryProfile}", null); + Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); + assertEquals("Profile: myProfile", q.properties().get("myField")); + } + + @Test public void testTimeoutInRequestOverridesQueryProfile() { QueryProfile profile = new QueryProfile("test"); profile.set("timeout", 318, (QueryProfileRegistry)null); @@ -332,6 +340,20 @@ public class QueryTestCase { } @Test + public void testQueryProfileInSubstitution() { + QueryProfile testProfile = new QueryProfile("test"); + testProfile.setOverridable("u", false, null); + testProfile.set("d","e", null); + testProfile.set("u","11", null); + testProfile.set("foo.bar", "wiz", null); + Query q = new Query(QueryTestCase.httpEncode("?query=a:>5&a=b&traceLevel=5&sources=a,b&u=12&foo.bar2=wiz2&c.d=foo&queryProfile=test"),testProfile.compile(null)); + String trace = q.getContext(false).getTrace().toString(); + String[] traceLines = trace.split("\n"); + for (String line : traceLines) + System.out.println(line); + } + + @Test public void testDefaultIndex() { Query q = new Query("?query=hi%20hello%20keyword:kanoo%20" + "default:munkz%20%22phrases+too%22&default-index=def"); @@ -385,18 +407,15 @@ public class QueryTestCase { } public class TestClass { + private int testInt = 0; - public int getTestInt() { - return testInt; - } - public void setTestInt(int testInt) { - this.testInt = testInt; - } + public int getTestInt() { return testInt; } + + public void setTestInt(int testInt) { this.testInt = testInt; } + + public void setTestInt(String testInt) { this.testInt = Integer.parseInt(testInt); } - public void setTestInt(String testInt) { - this.testInt = Integer.parseInt(testInt); - } } @Test @@ -431,7 +450,6 @@ public class QueryTestCase { Set<String> traces = new HashSet<>(); for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) traces.add(trace); - // for (String s : traces) System.out.println(s); assertTrue(traces.contains("trace1: [select * from sources * where default contains \"foo\";]")); assertTrue(traces.contains("trace2")); assertTrue(traces.contains("trace3-1, trace3-2: [select * from sources * where default contains \"foo\";]")); @@ -444,9 +462,8 @@ public class QueryTestCase { assertEquals(2, q.getTraceLevel()); q.trace(false,2, "trace2 ", null); Set<String> traces = new HashSet<>(); - for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) { + for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) traces.add(trace); - } assertTrue(traces.contains("trace2 null")); } @@ -460,8 +477,6 @@ public class QueryTestCase { Query q = new Query(QueryTestCase.httpEncode("?query=a:>5&a=b&traceLevel=5&sources=a,b&u=12&foo.bar2=wiz2&c.d=foo&queryProfile=test"),testProfile.compile(null)); String trace = q.getContext(false).getTrace().toString(); String[] traceLines = trace.split("\n"); - for (String line : traceLines) - System.out.println(line); assertTrue(contains("query=a:>5 (value from request)", traceLines)); assertTrue(contains("traceLevel=5 (value from request)", traceLines)); assertTrue(contains("a=b (value from request)", traceLines)); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index 1c8b861ba51..b1dad7d814e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; +import com.yahoo.component.Version; + +import java.time.Instant; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; @@ -17,7 +20,8 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { * Used in cases where application version cannot be determined, such as manual deployments (e.g. in dev * environment) */ - public static final ApplicationVersion unknown = new ApplicationVersion(Optional.empty(), OptionalLong.empty(), Optional.empty()); + public static final ApplicationVersion unknown = new ApplicationVersion(Optional.empty(), OptionalLong.empty(), + Optional.empty(), Optional.empty(), Optional.empty()); // This never changes and is only used to create a valid semantic version number, as required by application bundles private static final String majorVersion = "1.0"; @@ -25,33 +29,53 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { private final Optional<SourceRevision> source; private final Optional<String> authorEmail; private final OptionalLong buildNumber; + private final Optional<Version> compileVersion; + private final Optional<Instant> buildTime; - private ApplicationVersion(Optional<SourceRevision> source, OptionalLong buildNumber, Optional<String> authorEmail) { + private ApplicationVersion(Optional<SourceRevision> source, OptionalLong buildNumber, Optional<String> authorEmail, + Optional<Version> compileVersion, Optional<Instant> buildTime) { Objects.requireNonNull(source, "source cannot be null"); Objects.requireNonNull(buildNumber, "buildNumber cannot be null"); Objects.requireNonNull(authorEmail, "author cannot be null"); if (source.isPresent() != buildNumber.isPresent()) { throw new IllegalArgumentException("both buildNumber and source must be set together"); } + if (compileVersion.isPresent() != buildTime.isPresent()) { + throw new IllegalArgumentException("both compileVersion and buildTime must be set together"); + } if (buildNumber.isPresent() && buildNumber.getAsLong() <= 0) { throw new IllegalArgumentException("buildNumber must be > 0"); } if (authorEmail.isPresent() && ! authorEmail.get().matches("[^@]+@[^@]+")) { throw new IllegalArgumentException("Invalid author email '" + authorEmail.get() + "'."); } + if (compileVersion.isPresent() && compileVersion.get().equals(Version.emptyVersion)) { + throw new IllegalArgumentException("The empty version is not a legal compile version."); + } this.source = source; this.buildNumber = buildNumber; this.authorEmail = authorEmail; + this.compileVersion = compileVersion; + this.buildTime = buildTime; } /** Create an application package version from a completed build, without an author email */ public static ApplicationVersion from(SourceRevision source, long buildNumber) { - return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.empty()); + return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.empty(), + Optional.empty(), Optional.empty()); } - /** Create an application package version from a completed build, with an author email */ + /** Creates an version from a completed build and an author email. */ public static ApplicationVersion from(SourceRevision source, long buildNumber, String authorEmail) { - return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.of(authorEmail)); + return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), + Optional.of(authorEmail), Optional.empty(), Optional.empty()); + } + + /** Creates an version from a completed build, an author email, and build meta data. */ + public static ApplicationVersion from(SourceRevision source, long buildNumber, String authorEmail, + Version compileVersion, Instant buildTime) { + return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.of(authorEmail), + Optional.of(compileVersion), Optional.of(buildTime)); } /** Returns an unique identifier for this version or "unknown" if version is not known */ @@ -74,6 +98,12 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { /** Returns the email of the author of commit of this version, if known */ public Optional<String> authorEmail() { return authorEmail; } + /** Returns the Vespa version this package was compiled against, if known. */ + public Optional<Version> compileVersion() { return compileVersion; } + + /** Returns the time this package was built, if known. */ + public Optional<Instant> buildTime() { return buildTime; } + /** Returns whether this is unknown */ public boolean isUnknown() { return this.equals(unknown); @@ -86,19 +116,23 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { ApplicationVersion that = (ApplicationVersion) o; return Objects.equals(source, that.source) && Objects.equals(authorEmail, that.authorEmail) && - Objects.equals(buildNumber, that.buildNumber); + Objects.equals(buildNumber, that.buildNumber) && + Objects.equals(compileVersion, that.compileVersion) && + Objects.equals(buildTime, that.buildTime); } @Override public int hashCode() { - return Objects.hash(source, authorEmail, buildNumber); + return Objects.hash(source, authorEmail, buildNumber, compileVersion, buildTime); } @Override public String toString() { - return "Application package version: " + id() + return "Application package version: " + id() + source.map(s -> ", " + s.toString()).orElse("") - + authorEmail.map(e -> ", " + e).orElse(""); + + authorEmail.map(e -> ", by " + e).orElse("") + + compileVersion.map(v -> ", built against " + v).orElse("") + + buildTime.map(t -> " at " + t).orElse("") ; } /** Abbreviate given commit hash to 9 characters */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index 40e2e4a92d1..a0b4a888727 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -1,14 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; import org.apache.commons.codec.digest.DigestUtils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -27,10 +32,12 @@ public class ApplicationPackage { private final byte[] zippedContent; private final DeploymentSpec deploymentSpec; private final ValidationOverrides validationOverrides; + private final Optional<Version> compileVersion; + private final Optional<Instant> buildTime; /** * Creates an application package from its zipped content. - * This <b>assigns ownership</b> of the given byte array to this class: + * This <b>assigns ownership</b> of the given byte array to this class; * it must not be further changed by the caller. */ public ApplicationPackage(byte[] zippedContent) { @@ -38,6 +45,10 @@ public class ApplicationPackage { this.contentHash = DigestUtils.shaHex(zippedContent); this.deploymentSpec = extractFile("deployment.xml", zippedContent).map(DeploymentSpec::fromXml).orElse(DeploymentSpec.empty); this.validationOverrides = extractFile("validation-overrides.xml", zippedContent).map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); + Optional<Inspector> buildMetaObject = extractFileBytes("build-meta.json", zippedContent) + .map(SlimeUtils::jsonToSlime).map(Slime::get); + this.compileVersion = buildMetaObject.map(object -> Version.fromString(object.field("compileVersion").asString())); + this.buildTime = buildMetaObject.map(object -> Instant.ofEpochMilli(object.field("buildTime").asLong())); } /** Returns a hash of the content of this package */ @@ -58,12 +69,18 @@ public class ApplicationPackage { */ public ValidationOverrides validationOverrides() { return validationOverrides; } - private static Optional<Reader> extractFile(String fileName, byte[] zippedContent) { + /** Returns the platform version which package was compiled against, if known. */ + public Optional<Version> compileVersion() { return compileVersion; } + + /** Returns the time this package was built, if known. */ + public Optional<Instant> buildTime() { return buildTime; } + + private static Optional<byte[]> extractFileBytes(String fileName, byte[] zippedContent) { try (ByteArrayInputStream stream = new ByteArrayInputStream(zippedContent)) { ZipStreamReader reader = new ZipStreamReader(stream); for (ZipStreamReader.ZipEntryWithContent entry : reader.entries()) if (entry.zipEntry().getName().equals(fileName) || entry.zipEntry().getName().equals("application/" + fileName)) // TODO: Remove application/ directory support - return Optional.of(new InputStreamReader(new ByteArrayInputStream(entry.content()))); + return Optional.of(entry.content()); return Optional.empty(); } catch (IOException e) { @@ -71,4 +88,8 @@ public class ApplicationPackage { } } + private static Optional<Reader> extractFile(String fileName, byte[] zippedContent) { + return extractFileBytes(fileName, zippedContent).map(ByteArrayInputStream::new).map(InputStreamReader::new); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 14222ca96d2..81a4e30feac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -504,7 +504,7 @@ public class InternalStepRunner implements StepRunner { DeploymentSpec spec = controller.applications().require(id.application()).deploymentSpec(); ZoneId zone = id.type().zone(controller.system()); - byte[] deploymentXml = deploymentXml(spec.athenzDomain().get(), spec.athenzService(zone.environment(), zone.region()).get()); + byte[] deploymentXml = deploymentXml(spec.athenzDomain(), spec.athenzService(zone.environment(), zone.region())); try (ZipBuilder zipBuilder = new ZipBuilder(testPackage.length + servicesXml.length + 1000)) { zipBuilder.add(testPackage); @@ -591,11 +591,14 @@ public class InternalStepRunner implements StepRunner { return servicesXml.getBytes(); } - /** Returns a dummy deployment xml which sets up the service identity for the tester. */ - static byte[] deploymentXml(AthenzDomain domain, AthenzService service) { + /** Returns a dummy deployment xml which sets up the service identity for the tester, if present. */ + private static byte[] deploymentXml(Optional<AthenzDomain> athenzDomain, Optional<AthenzService> athenzService) { String deploymentSpec = "<?xml version='1.0' encoding='UTF-8'?>\n" + - "<deployment version=\"1.0\" athenz-domain=\"" + domain.value() + "\" athenz-service=\"" + service.value() + "\" />"; + "<deployment version=\"1.0\" " + + athenzDomain.map(domain -> "athenz-domain=\"" + domain.value() + "\" ").orElse("") + + athenzService.map(service -> "athenz-service=\"" + service.value() + "\" ").orElse("") + + "/>"; return deploymentSpec.getBytes(StandardCharsets.UTF_8); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 5c56c7d2280..fcead86893a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -10,25 +10,23 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.net.URI; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.Deque; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -41,7 +39,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -229,7 +226,7 @@ public class JobController { * Accepts and stores a new application package and test jar pair under a generated application version key. */ public ApplicationVersion submit(ApplicationId id, SourceRevision revision, String authorEmail, long projectId, - byte[] packageBytes, byte[] testPackageBytes) { + ApplicationPackage applicationPackage, byte[] testPackageBytes) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { if ( ! application.get().deploymentJobs().deployedInternally()) { @@ -245,17 +242,22 @@ public class JobController { } long run = nextBuild(id); - version.set(ApplicationVersion.from(revision, run, authorEmail)); + if (applicationPackage.compileVersion().isPresent() && applicationPackage.buildTime().isPresent()) + version.set(ApplicationVersion.from(revision, run, authorEmail, + applicationPackage.compileVersion().get(), + applicationPackage.buildTime().get())); + else + version.set(ApplicationVersion.from(revision, run, authorEmail)); controller.applications().applicationStore().put(id, version.get(), - packageBytes); + applicationPackage.zippedContent()); controller.applications().applicationStore().put(TesterId.of(id), version.get(), testPackageBytes); prunePackages(id); - controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), new ApplicationPackage(packageBytes)); + controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), applicationPackage); controller.applications().deploymentTrigger().notifyOfCompletion(DeploymentJobs.JobReport.ofSubmission(id, projectId, version.get())); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 7924ea34475..08e5be2ea1f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -72,7 +72,7 @@ public class ControllerMaintenance extends AbstractComponent { readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofMinutes(1), jobControl); clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient); clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); - deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); + deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 305241a7d0e..2ae38fabb94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; @@ -37,6 +38,7 @@ public class MetricsReporter extends Maintainer { public static final String deploymentFailMetric = "deployment.failurePercentage"; public static final String deploymentAverageDuration = "deployment.averageDuration"; public static final String deploymentFailingUpgrades = "deployment.failingUpgrades"; + public static final String deploymentBuildAgeSeconds = "deployment.buildAgeSeconds"; public static final String remainingRotations = "remaining_rotations"; private final Metric metric; @@ -128,6 +130,14 @@ public class MetricsReporter extends Maintainer { deploymentsFailingUpgrade(applications).forEach((application, failingJobs) -> { metric.set(deploymentFailingUpgrades, failingJobs, metric.createContext(dimensions(application))); }); + + for (Application application : applications.asList()) + application.deploymentJobs().statusOf(JobType.component) + .flatMap(JobStatus::lastSuccess) + .flatMap(run -> run.application().buildTime()) + .ifPresent(buildTime -> metric.set(deploymentBuildAgeSeconds, + controller().clock().instant().getEpochSecond() - buildTime.getEpochSecond(), + metric.createContext(dimensions(application.id())))); } private static double deploymentFailRatio(ApplicationList applicationList) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 69ea36c7e3a..147b2edee3e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -81,6 +81,8 @@ public class ApplicationSerializer { private final String branchField = "branchField"; private final String commitField = "commitField"; private final String authorEmailField = "authorEmailField"; + private final String compileVersionField = "compileVersion"; + private final String buildTimeField = "buildTime"; private final String lastQueriedField = "lastQueried"; private final String lastWrittenField = "lastWritten"; private final String lastQueriesPerSecondField = "lastQueriesPerSecond"; @@ -231,6 +233,8 @@ public class ApplicationSerializer { object.setLong(applicationBuildNumberField, applicationVersion.buildNumber().getAsLong()); toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField)); applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email)); + applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString())); + applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli())); } } @@ -406,12 +410,21 @@ public class ApplicationSerializer { if ( ! object.valid()) return ApplicationVersion.unknown; OptionalLong applicationBuildNumber = optionalLong(object.field(applicationBuildNumberField)); Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); - if (!sourceRevision.isPresent() || !applicationBuildNumber.isPresent()) { + if ( ! sourceRevision.isPresent() || ! applicationBuildNumber.isPresent()) { return ApplicationVersion.unknown; } - return object.field(authorEmailField).valid() - ? ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), object.field(authorEmailField).asString()) - : ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong()); + Optional<String> authorEmail = optionalString(object.field(authorEmailField)); + Optional<Version> compileVersion = optionalString(object.field(compileVersionField)).map(Version::fromString); + Optional<Instant> buildTime = optionalInstant(object.field(buildTimeField)); + + if ( ! authorEmail.isPresent()) + return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong()); + + if ( ! compileVersion.isPresent() || ! buildTime.isPresent()) + return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get()); + + return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get(), + compileVersion.get(), buildTime.get()); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index ef91a2e5a15..32c92e6f135 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -68,6 +68,8 @@ class RunSerializer { private static final String branchField = "branch"; private static final String commitField = "commit"; private static final String authorEmailField = "authorEmail"; + private static final String compileVersionField = "compileVersion"; + private static final String buildTimeField = "buildTime"; private static final String buildField = "build"; private static final String sourceField = "source"; private static final String lastTestRecordField = "lastTestRecord"; @@ -123,9 +125,16 @@ class RunSerializer { versionObject.field(branchField).asString(), versionObject.field(commitField).asString()); long buildNumber = versionObject.field(buildField).asLong(); - return versionObject.field(authorEmailField).valid() - ? ApplicationVersion.from(revision, buildNumber, versionObject.field(authorEmailField).asString()) - : ApplicationVersion.from(revision, buildNumber); + + if ( ! versionObject.field(authorEmailField).valid()) + return ApplicationVersion.from(revision, buildNumber); + + if ( ! versionObject.field(compileVersionField).valid() || ! versionObject.field(buildTimeField).valid()) + return ApplicationVersion.from(revision, buildNumber, versionObject.field(authorEmailField).asString()); + + return ApplicationVersion.from(revision, buildNumber, versionObject.field(authorEmailField).asString(), + Version.fromString(versionObject.field(compileVersionField).asString()), + Instant.ofEpochMilli(versionObject.field(buildTimeField).asLong())); } Slime toSlime(Iterable<Run> runs) { @@ -173,6 +182,8 @@ class RunSerializer { versionsObject.setLong(buildField, applicationVersion.buildNumber() .orElseThrow(() -> new IllegalArgumentException("Build number must be present in application version."))); applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email)); + applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString())); + applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli())); } static String valueOf(Step step) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index c8104c6e2b0..56e1e746d04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -803,7 +803,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Change change = application.get().change(); - if ( ! change.isPresent()) { + if ( ! change.isPresent() && ! change.isPinned()) { response.append("No deployment in progress for " + application + " at this time"); return; } @@ -1322,9 +1322,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { long projectId = Math.max(1, submitOptions.field("projectId").asLong()); ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); - if ( ! applicationPackage.deploymentSpec().athenzDomain().isPresent()) - throw new IllegalArgumentException("Application must define an Athenz service in deployment.xml!"); - controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), applicationPackage, Optional.of(getUserPrincipal(request).getIdentity())); + controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), + applicationPackage, + Optional.of(getUserPrincipal(request).getIdentity())); return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, @@ -1332,7 +1332,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { sourceRevision, authorEmail, projectId, - applicationPackage.zippedContent(), + applicationPackage, dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index a3dc3f1e706..7d49ffe8139 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobStatus; @@ -27,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -398,19 +400,16 @@ class JobControllerApiHandlerHelper { * @return Response with the new application version */ static HttpResponse submitResponse(JobController jobController, String tenant, String application, - SourceRevision sourceRevision, String authorEmail, - long projectId, byte[] appPackage, byte[] testPackage) { + SourceRevision sourceRevision, String authorEmail, long projectId, + ApplicationPackage applicationPackage, byte[] testPackage) { ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, "default"), sourceRevision, authorEmail, projectId, - appPackage, + applicationPackage, testPackage); - Slime slime = new Slime(); - Cursor responseObject = slime.setObject(); - responseObject.setString("version", version.id()); - return new SlimeJsonResponse(slime); + return new MessageResponse(version.toString()); } /** Aborts any job of the given type. */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index b7a881c672e..1199f0229b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -155,7 +155,8 @@ public class ControllerTest { .region("deep-space-9") .build(); try { - tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, "a@b", 2, applicationPackage.zippedContent(), new byte[0]); + tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, "a@b", + 2, applicationPackage, new byte[0]); fail("Expected exception due to illegal deployment spec."); } catch (IllegalArgumentException e) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 0499a92d05f..44a797687d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -170,6 +170,10 @@ public class ApplicationPackageBuilder { return searchDefinition.getBytes(StandardCharsets.UTF_8); } + private byte[] buildMeta() { + return "{\"compileVersion\":\"6.1\",\"buildTime\":1000}".getBytes(StandardCharsets.UTF_8); + } + public ApplicationPackage build() { ByteArrayOutputStream zip = new ByteArrayOutputStream(); try (ZipOutputStream out = new ZipOutputStream(zip)) { @@ -182,6 +186,9 @@ public class ApplicationPackageBuilder { out.putNextEntry(new ZipEntry("search-definitions/test.sd")); out.write(searchDefinition()); out.closeEntry(); + out.putNextEntry(new ZipEntry("build-meta.json")); + out.write(buildMeta()); + out.closeEntry(); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index 80cbbfa2c30..35e7aea5efe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -89,7 +89,7 @@ public class InternalDeploymentTester { * Submits a new application, and returns the version of the new submission. */ public ApplicationVersion newSubmission() { - ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, "a@b", 2, applicationPackage.zippedContent(), new byte[0]); + ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, "a@b", 2, applicationPackage, new byte[0]); tester.applicationStore().put(appId, version, applicationPackage.zippedContent()); tester.applicationStore().put(testerId, version, new byte[0]); return version; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 7e6e892891a..8ebb8c108c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -64,6 +65,7 @@ import static org.junit.Assert.fail; */ public class JobRunnerTest { + private static final ApplicationPackage applicationPackage = new ApplicationPackage(new byte[0]); private static final Versions versions = new Versions(Version.fromString("1.2.3"), ApplicationVersion.from(new SourceRevision("repo", "branch", @@ -82,7 +84,7 @@ public class JobRunnerTest { phasedExecutor(phaser), stepRunner); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); try { @@ -113,7 +115,7 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); jobs.start(id, systemTest, versions); @@ -197,7 +199,7 @@ public class JobRunnerTest { Executors.newFixedThreadPool(32), waitingRunner(barrier)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); RunId runId = new RunId(id, systemTest, 1); jobs.start(id, systemTest, versions); @@ -233,7 +235,7 @@ public class JobRunnerTest { inThreadExecutor(), (id, step) -> Optional.of(running)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); for (int i = 0; i < jobs.historyLength(); i++) { jobs.start(id, systemTest, versions); @@ -261,7 +263,7 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), "a@b", 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 7ab858f3081..9c01e526a50 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -11,9 +11,11 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.MetricsMock.MapContext; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -204,6 +206,18 @@ public class MetricsReporterTest { assertEquals(0, getDeploymentsFailingUpgrade(app)); } + @Test + public void testBuildTimeReporting() { + InternalDeploymentTester tester = new InternalDeploymentTester(); + ApplicationVersion version = tester.deployNewSubmission(); + assertEquals(1000, version.buildTime().get().toEpochMilli()); + + MetricsReporter reporter = createReporter(tester.tester().controller(), metrics, SystemName.main); + reporter.maintain(); + assertEquals(tester.clock().instant().getEpochSecond() - 1, + getMetric(MetricsReporter.deploymentBuildAgeSeconds, tester.app())); + } + private Duration getAverageDeploymentDuration(Application application) { return Duration.ofSeconds(getMetric(MetricsReporter.deploymentAverageDuration, application).longValue()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 774caea97b0..0b337eb5380 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -10,11 +10,12 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -25,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import org.junit.Test; @@ -70,7 +70,8 @@ public class ApplicationSerializerTest { List<Deployment> deployments = new ArrayList<>(); ApplicationVersion applicationVersion1 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 31); ApplicationVersion applicationVersion2 = ApplicationVersion - .from(new SourceRevision("repo1", "branch1", "commit1"), 32, "a@b"); + .from(new SourceRevision("repo1", "branch1", "commit1"), 32, "a@b", + Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index d6334a9ea86..03b432588bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -19,6 +19,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.Optional; @@ -78,7 +79,9 @@ public class RunSerializerTest { "master", "f00bad"), 123, - "a@b"), + "a@b", + Version.fromString("6.3.1"), + Instant.ofEpochMilli(100)), run.versions().targetApplication()); assertEquals(new Version(1, 2, 2), run.versions().sourcePlatform().get()); assertEquals(ApplicationVersion.from(new SourceRevision("git@github.com:user/repo.git", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 75bbea6861d..cda3834d47d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -26,6 +26,8 @@ "commit": "f00bad", "build": 123, "authorEmail": "a@b", + "compileVersion": "6.3.1", + "buildTime": 100, "source": { "platform": "1.2.2", "repository": "git@github.com:user/repo.git", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index d455218f4e9..30795008032 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -361,24 +361,40 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE (cancel) ongoing change tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) .userIdentity(HOSTED_VESPA_OPERATOR), - new File("application-deployment-cancelled.json")); + "{\"message\":\"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1'\"}"); // DELETE (cancel) again is a no-op tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) .userIdentity(USER_ID) .data("{\"cancel\":\"all\"}"), - new File("application-deployment-cancelled-no-op.json")); + "{\"message\":\"No deployment in progress for application 'tenant1.application1' at this time\"}"); // POST pinning to a given version to an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) .userIdentity(USER_ID) .data("6.1.0"), - new File("application-deployment.json")); + "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); // DELETE only the pin to a given version tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) .userIdentity(USER_ID), - new File("application-pin-cancelled.json")); + "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1'\"}"); + + // POST pinning again + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) + .userIdentity(USER_ID) + .data("6.1"), + "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); + + // DELETE only the version, but leave the pin + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/platform", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for application 'tenant1.application1'\"}"); + + // DELETE also the pin to a given version + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Changed deployment from 'pin to current platform' to 'no change' for application 'tenant1.application1'\"}"); // POST a pause to a production job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1/pause", POST) @@ -435,13 +451,13 @@ public class ApplicationApiTest extends ControllerContainerTest { "Deactivated tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); // POST an application package and a test jar, submitting a new application for internal pipeline deployment. - // First attempt does not have an Athenz service definition in deployment spec, and fails. + // First attempt does not have an Athenz service definition in deployment spec, and is accepted. tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(applicationPackage)), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Application must define an Athenz service in deployment.xml!\"}", 400); + "{\"message\":\"Application package version: 1.0.43-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); - // Second attempt has a service under a different domain than the tenant of the application, and fails as well. + // Second attempt has a service under a different domain than the tenant of the application, and fails. ApplicationPackage packageWithServiceForWrongDomain = new ApplicationPackageBuilder() .environment(Environment.prod) .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN_2.getName()), AthenzService.from("service")) @@ -461,7 +477,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(packageWithService)), - "{\"version\":\"1.0.43-d00d\"}"); + "{\"message\":\"Application package version: 1.0.44-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/badge", GET) .userIdentity(USER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json deleted file mode 100644 index 91d3e64d6db..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json +++ /dev/null @@ -1 +0,0 @@ -{"message":"No deployment in progress for application 'tenant1.application1' at this time"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json deleted file mode 100644 index efca5831256..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json +++ /dev/null @@ -1 +0,0 @@ -{"message":"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1'"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json deleted file mode 100644 index fe68f3d94a3..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json +++ /dev/null @@ -1 +0,0 @@ -{"message":"Triggered pin to 6.1 for tenant1.application1"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json deleted file mode 100644 index 62360458ce4..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json +++ /dev/null @@ -1 +0,0 @@ -{"message":"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1'"} diff --git a/document/abi-spec.json b/document/abi-spec.json index 2b3d443d02e..79e0cdc34d0 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -2463,6 +2463,7 @@ "public void clear()", "public void assign(java.lang.Object)", "public boolean getBoolean()", + "public void setBoolean(boolean)", "public java.lang.Object getWrappedValue()", "public com.yahoo.document.DataType getDataType()", "public void printXml(com.yahoo.document.serialization.XmlStream)", @@ -4740,6 +4741,7 @@ "public abstract void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", "public abstract void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", "public abstract void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", + "public abstract void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", "public abstract void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", "public abstract void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", "public abstract void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", @@ -4773,6 +4775,7 @@ "public abstract void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", "public abstract void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", "public abstract void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", + "public abstract void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", "public abstract void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", "public abstract void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", "public abstract void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", @@ -4878,6 +4881,7 @@ "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", @@ -4917,7 +4921,8 @@ ], "methods": [ "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", - "public void read(com.yahoo.document.DocumentUpdate)" + "public void read(com.yahoo.document.DocumentUpdate)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)" ], "fields": [] }, @@ -4937,6 +4942,7 @@ "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", @@ -4982,7 +4988,8 @@ "public void write(com.yahoo.document.DocumentUpdate)", "public void write(com.yahoo.document.fieldpathupdate.FieldPathUpdate)", "public void write(com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate)", - "public void write(com.yahoo.document.fieldpathupdate.AddFieldPathUpdate)" + "public void write(com.yahoo.document.fieldpathupdate.AddFieldPathUpdate)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)" ], "fields": [] }, @@ -5003,6 +5010,7 @@ "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", diff --git a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java index 2a48b550658..189c275809a 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java @@ -60,6 +60,7 @@ public class BoolFieldValue extends FieldValue { public boolean getBoolean() { return value; } + public void setBoolean(boolean value) { this.value = value; } @Override public Object getWrappedValue() { diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index 6adae27cadc..e4ca20a8e32 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -9,6 +9,7 @@ import com.yahoo.document.DocumentUpdate; import com.yahoo.document.FieldPath; import com.yahoo.document.annotation.AnnotationReference; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; @@ -285,6 +286,11 @@ public class DocumentUpdateJsonSerializer } @Override + public void write(FieldBase field, BoolFieldValue value) { + serializeBoolField(generator, field, value); + } + + @Override public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { serializeCollectionField(this, generator, field, value); } diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index afe14cf1e6a..55e7dc3c1b3 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -8,6 +8,7 @@ import com.yahoo.document.Field; import com.yahoo.document.PositionDataType; import com.yahoo.document.PrimitiveDataType; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; @@ -234,6 +235,10 @@ public class JsonSerializationHelper { serializeByte(generator, field, value.getByte()); } + public static void serializeBoolField(JsonGenerator generator, FieldBase field, BoolFieldValue value) { + serializeBool(generator, field, value.getBoolean()); + } + public static void serializePredicateField(JsonGenerator generator, FieldBase field, PredicateFieldValue value){ serializeString(generator, field, value.toString()); } @@ -252,6 +257,11 @@ public class JsonSerializationHelper { wrapIOException(() -> generator.writeNumber(value)); } + public static void serializeBool(JsonGenerator generator, FieldBase field, boolean value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeBoolean(value)); + } + public static void serializeShort(JsonGenerator generator, FieldBase field, short value) { fieldNameIfNotNull(generator, field); wrapIOException(() -> generator.writeNumber(value)); diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index ecec34c5d3f..ab0884a54a3 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -9,6 +9,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.annotation.AnnotationReference; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; @@ -140,6 +141,11 @@ public class JsonWriter implements DocumentWriter { } @Override + public void write(FieldBase field, BoolFieldValue value) { + serializeBoolField(generator, field, value); + } + + @Override public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { serializeCollectionField(this, generator, field, value); } diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldReader.java b/document/src/main/java/com/yahoo/document/serialization/FieldReader.java index 11fc0c314af..0b1500ed6ba 100644 --- a/document/src/main/java/com/yahoo/document/serialization/FieldReader.java +++ b/document/src/main/java/com/yahoo/document/serialization/FieldReader.java @@ -6,7 +6,24 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; import com.yahoo.document.annotation.AnnotationReference; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.ReferenceFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.vespa.objects.Deserializer; import com.yahoo.vespa.objects.FieldBase; @@ -54,6 +71,14 @@ public interface FieldReader extends Deserializer { void read(FieldBase field, ByteFieldValue value); /** + * Read in the value of byte field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, BoolFieldValue value); + + /** * Read in the value of collection field * * @param field - field description (name and data type) diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java index 243d25c3950..63a6d997b04 100644 --- a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java @@ -4,6 +4,7 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; import com.yahoo.document.annotation.AnnotationReference; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; @@ -78,6 +79,16 @@ public interface FieldWriter extends Serializer { void write(FieldBase field, ByteFieldValue value); /** + * Write out the value of byte field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, BoolFieldValue value); + + /** * Write out the value of collection field * * @param field diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java index 6ec7a1e2b21..7ff5729ca39 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -27,6 +27,7 @@ import com.yahoo.document.annotation.SpanNode; import com.yahoo.document.annotation.SpanNodeParent; import com.yahoo.document.annotation.SpanTree; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; @@ -111,6 +112,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple public void read(Document document) { read(null, document); } + public void read(FieldBase field, Document doc) { // Verify that we have correct version @@ -219,6 +221,10 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple public <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value) { throw new IllegalArgumentException("read not implemented yet."); } + @Override + public void read(FieldBase field, BoolFieldValue value) { + value.setBoolean((getByte(null) != 0)); + } public void read(FieldBase field, ByteFieldValue value) { value.assign(getByte(null)); } public void read(FieldBase field, DoubleFieldValue value) { value.assign(getDouble(null)); } public void read(FieldBase field, FloatFieldValue value) { value.assign(getFloat(null)); } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java index 44a1ca6e749..40aec94aec6 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java @@ -4,9 +4,11 @@ package com.yahoo.document.serialization; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.fieldpathupdate.FieldPathUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.vespa.objects.FieldBase; /** * Class used for de-serializing documents on the current head document format. @@ -42,4 +44,8 @@ public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer42 { } } + @Override + public void read(FieldBase field, BoolFieldValue value) { + value.setBoolean((getByte(null) != 0)); + } } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java index 6a07e04a621..581c7df8aee 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java @@ -20,6 +20,7 @@ import com.yahoo.document.annotation.SpanList; import com.yahoo.document.annotation.SpanNode; import com.yahoo.document.annotation.SpanTree; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; @@ -57,7 +58,6 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.logging.Logger; import static com.yahoo.text.Utf8.calculateBytePositions; @@ -168,16 +168,17 @@ public class VespaDocumentSerializer42 extends BufferSerializer implements Docum } } - /** - * Write out the value of byte field - * - * @param field - field description (name and data type) - * @param value - field value - */ + @Override public void write(FieldBase field, ByteFieldValue value) { buf.put(value.getByte()); } + @Override + public void write(FieldBase field, BoolFieldValue value) { + byte v = value.getBoolean() ? (byte)1 : (byte)0; + buf.put(v); + } + /** * Write out the value of collection field * diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java index ae995371125..92bce41ba8c 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java @@ -2,11 +2,13 @@ package com.yahoo.document.serialization; import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; import com.yahoo.document.fieldpathupdate.FieldPathUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.vespa.objects.FieldBase; /** * Class used for serializing documents on the current head document format. @@ -70,4 +72,9 @@ public class VespaDocumentSerializerHead extends VespaDocumentSerializer42 { write((FieldPathUpdate)update); update.getNewValues().serialize(this); } + + @Override + public void write(FieldBase field, ByteFieldValue value) { + buf.put(value.getByte()); + } } diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index 768ec879ce1..0d6b0cae926 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -7,6 +7,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.annotation.AnnotationReference; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; @@ -137,6 +138,11 @@ public final class XmlDocumentWriter implements DocumentWriter { } @Override + public void write(FieldBase field, BoolFieldValue value) { + singleValueTag(field, value); + } + + @Override public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { buffer.beginTag(field.getName()); diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java index 8aa34ae9bba..737371f2375 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java @@ -298,6 +298,19 @@ public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader { } } + public void read(FieldBase field, BoolFieldValue value) { + try { + String dataParsed = reader.getElementText(); + try { + value.assign(dataParsed); + } catch (Exception e) { + throw newDeserializeException(field, e.getMessage()); + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + public void read(FieldBase field, DoubleFieldValue value) { try { String dataParsed = reader.getElementText(); diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index e45da62353d..7d1992225e4 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.document; import com.yahoo.compress.CompressionType; import com.yahoo.document.annotation.AbstractTypesTest; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; import com.yahoo.document.datatypes.FloatFieldValue; @@ -72,6 +73,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { docType.addField(new Field("rawfield", DataType.RAW, false)); docType.addField(new Field("doublefield", DataType.DOUBLE, false)); docType.addField(new Field("bytefield", DataType.BYTE, false)); + docType.addField(new Field("boolfield", DataType.BOOL, false)); DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT); docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false)); DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType); @@ -94,6 +96,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { doc.setFieldValue("longfield", new LongFieldValue(398420092938472983l)); doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820)); doc.setFieldValue("bytefield", new ByteFieldValue(254)); + doc.setFieldValue("boolfield", new BoolFieldValue(true)); byte[] rawData = "RAW DATA".getBytes(); assertEquals(8, rawData.length); doc.setFieldValue(docType.getField("rawfield"),new Raw(ByteBuffer.wrap("RAW DATA".getBytes()))); @@ -176,12 +179,12 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield")); assertEquals(new LongFieldValue(398420092938472983l), doc.getFieldValue("longfield")); assertEquals(98374532.398820, ((DoubleFieldValue)doc.getFieldValue("doublefield")).getDouble(), 1E-6); - assertEquals(new ByteFieldValue((byte)254), - doc.getFieldValue("bytefield")); + assertEquals(new ByteFieldValue((byte)254), doc.getFieldValue("bytefield")); + // Todo add cpp serialization + // assertEquals(new BoolFieldValue(true), doc.getFieldValue("boolfield")); ByteBuffer bbuffer = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer(); if (!Arrays.equals("RAW DATA".getBytes(), bbuffer.array())) { - System.err.println("Expected 'RAW DATA' but got '" - + new String(bbuffer.array()) + "'."); + System.err.println("Expected 'RAW DATA' but got '" + new String(bbuffer.array()) + "'."); assertTrue(false); } if (test.version > 6) { diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 6a2147d6f15..3eebc4396e8 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.compress.CompressionType; import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; import com.yahoo.document.datatypes.DoubleFieldValue; import com.yahoo.document.datatypes.FieldPathIteratorHandler; @@ -89,6 +90,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { " </item>\n" + " </mapfield>\n" + SERTEST_DOC_AS_XML_SUNNYVALE + + " <myboolfield>true</myboolfield>\n" + "</document>\n"; static DocumentTypeManager setUpCppDocType() { @@ -124,6 +126,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { sertestDocType.addField(new Field("docindoc", 882, docInDocType, false)); sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING), false)); sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE, false)); + sertestDocType.addField(new Field("myboolfield", 885, DataType.BOOL, false)); docMan.registerDocumentType(sertestDocType); } @@ -165,6 +168,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1")); map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2")); doc.setFieldValue("mapfield", map); + doc.setFieldValue("myboolfield", new BoolFieldValue(true)); return doc; } @@ -776,12 +780,12 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.setFieldValue("wsfield", wset); MapFieldValue<StringFieldValue, StringFieldValue> map = - new MapFieldValue<>( - (MapDataType)doc.getDataType().getField("mapfield").getDataType()); + new MapFieldValue<>((MapDataType)doc.getDataType().getField("mapfield").getDataType()); map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1")); map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2")); doc.setFieldValue("mapfield", map); + doc.setFieldValue("boolfield", new BoolFieldValue(true)); doc.setFieldValue("bytefield", new ByteFieldValue((byte)254)); doc.setFieldValue("rawfield", new Raw(ByteBuffer.wrap("RAW DATA".getBytes()))); doc.setFieldValue("intfield", new IntegerFieldValue(5)); @@ -841,6 +845,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { Document doc2 = docMan.createDocument(data); + assertEquals(doc.getFieldValue("myboolfield"), doc2.getFieldValue("myboolfield")); assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid")); assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date")); assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from")); @@ -1229,7 +1234,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { assertEquals(fields.get("date"), -2013512400); assertThat(fields.get("docindoc"), instanceOf(Map.class)); assertThat(fields.keySet(), - containsInAnyOrder("mailid", "date", "attachmentcount", "rawfield", "weightedfield", "docindoc", "mapfield")); + containsInAnyOrder("mailid", "date", "attachmentcount", "rawfield", "weightedfield", "docindoc", "mapfield", "myboolfield")); } } diff --git a/document/src/tests/data/crossplatform-java-cpp-document.cfg b/document/src/tests/data/crossplatform-java-cpp-document.cfg index f12dae77fc0..134d31b1831 100644 --- a/document/src/tests/data/crossplatform-java-cpp-document.cfg +++ b/document/src/tests/data/crossplatform-java-cpp-document.cfg @@ -83,7 +83,7 @@ datatype[8].weightedsettype[0] datatype[8].structtype[1] datatype[8].structtype[0].name serializetest.body datatype[8].structtype[0].version 0 -datatype[8].structtype[0].field[10] +datatype[8].structtype[0].field[11] datatype[8].structtype[0].field[0].name intfield datatype[8].structtype[0].field[0].id[0] datatype[8].structtype[0].field[0].datatype 0 @@ -114,6 +114,9 @@ datatype[8].structtype[0].field[8].datatype 437829 datatype[8].structtype[0].field[9].name mapfield datatype[8].structtype[0].field[9].id[0] datatype[8].structtype[0].field[9].datatype 9999 +datatype[8].structtype[0].field[10].name boolfield +datatype[8].structtype[0].field[10].id[0] +datatype[8].structtype[0].field[10].datatype 6 datatype[8].documenttype[0] datatype[9].id 1306012852 datatype[9].arraytype[0] diff --git a/document/src/tests/data/serializejava-compressed.dat b/document/src/tests/data/serializejava-compressed.dat Binary files differindex 453abef81f1..0f6cb55ff85 100644 --- a/document/src/tests/data/serializejava-compressed.dat +++ b/document/src/tests/data/serializejava-compressed.dat diff --git a/document/src/tests/data/serializejava.dat b/document/src/tests/data/serializejava.dat Binary files differindex 3dd9f8fcd52..53ef6a8fbc2 100644 --- a/document/src/tests/data/serializejava.dat +++ b/document/src/tests/data/serializejava.dat diff --git a/document/src/vespa/document/datatype/datatype.cpp b/document/src/vespa/document/datatype/datatype.cpp index 8d2721a4d9b..8c17ca4e383 100644 --- a/document/src/vespa/document/datatype/datatype.cpp +++ b/document/src/vespa/document/datatype/datatype.cpp @@ -21,6 +21,7 @@ NumericDataType INT_OBJ(DataType::T_INT); NumericDataType LONG_OBJ(DataType::T_LONG); NumericDataType FLOAT_OBJ(DataType::T_FLOAT); NumericDataType DOUBLE_OBJ(DataType::T_DOUBLE); +NumericDataType BOOL_OBJ(DataType::T_BOOL); PrimitiveDataType STRING_OBJ(DataType::T_STRING); PrimitiveDataType RAW_OBJ(DataType::T_RAW); DocumentType DOCUMENT_OBJ("document"); @@ -37,6 +38,7 @@ const DataType *const DataType::INT(&INT_OBJ); const DataType *const DataType::LONG(&LONG_OBJ); const DataType *const DataType::FLOAT(&FLOAT_OBJ); const DataType *const DataType::DOUBLE(&DOUBLE_OBJ); +const DataType *const DataType::BOOL(&BOOL_OBJ); const DataType *const DataType::STRING(&STRING_OBJ); const DataType *const DataType::RAW(&RAW_OBJ); const DocumentType *const DataType::DOCUMENT(&DOCUMENT_OBJ); @@ -71,6 +73,7 @@ DataType2FieldValueId::DataType2FieldValueId() _type2FieldValueId[DataType::T_LONG] = LongFieldValue::classId; _type2FieldValueId[DataType::T_FLOAT] = FloatFieldValue::classId; _type2FieldValueId[DataType::T_DOUBLE] = DoubleFieldValue::classId; + _type2FieldValueId[DataType::T_BOOL] = BoolFieldValue::classId; _type2FieldValueId[DataType::T_STRING] = StringFieldValue::classId; _type2FieldValueId[DataType::T_RAW] = RawFieldValue::classId; _type2FieldValueId[DataType::T_URI] = StringFieldValue::classId; @@ -103,6 +106,7 @@ DataType::getDefaultDataTypes() types.push_back(LONG); types.push_back(FLOAT); types.push_back(DOUBLE); + types.push_back(BOOL); types.push_back(STRING); types.push_back(RAW); types.push_back(DOCUMENT); diff --git a/document/src/vespa/document/datatype/datatype.h b/document/src/vespa/document/datatype/datatype.h index 4dd5d6aae64..723e7c69ed6 100644 --- a/document/src/vespa/document/datatype/datatype.h +++ b/document/src/vespa/document/datatype/datatype.h @@ -66,6 +66,7 @@ public: T_RAW = 3, T_LONG = 4, T_DOUBLE = 5, + T_BOOL = 6, T_DOCUMENT = 8, // Type of super document type Document.0 that all documents inherit. // T_TIMESTAMP = 9, // Not used anymore, Id should probably not be reused T_URI = 10, @@ -88,6 +89,7 @@ public: static const DataType *const LONG; static const DataType *const FLOAT; static const DataType *const DOUBLE; + static const DataType *const BOOL; static const DataType *const STRING; static const DataType *const RAW; static const DocumentType *const DOCUMENT; @@ -108,7 +110,7 @@ public: * Create a field value using this datatype. */ virtual std::unique_ptr<FieldValue> createFieldValue() const = 0; - virtual DataType* clone() const override = 0; + DataType* clone() const override = 0; /** * Whether another datatype is a supertype of this one. Document types may diff --git a/document/src/vespa/document/datatype/primitivedatatype.cpp b/document/src/vespa/document/datatype/primitivedatatype.cpp index e48e4464acf..7ec47f52d9c 100644 --- a/document/src/vespa/document/datatype/primitivedatatype.cpp +++ b/document/src/vespa/document/datatype/primitivedatatype.cpp @@ -23,6 +23,7 @@ namespace { const char *Double = "Double"; const char *Uri = "Uri"; const char *Byte = "Byte"; + const char *Bool = "Bool"; const char *Predicate = "Predicate"; const char *Tensor = "Tensor"; @@ -37,6 +38,7 @@ namespace { case DataType::T_DOUBLE: return Double; case DataType::T_URI: return Uri; case DataType::T_BYTE: return Byte; + case DataType::T_BOOL: return Bool; case DataType::T_PREDICATE: return Predicate; case DataType::T_TENSOR: return Tensor; default: @@ -56,16 +58,17 @@ FieldValue::UP PrimitiveDataType::createFieldValue() const { switch (getId()) { - case T_INT: return FieldValue::UP(new IntFieldValue); - case T_SHORT: return FieldValue::UP(new ShortFieldValue); - case T_FLOAT: return FieldValue::UP(new FloatFieldValue); - case T_URI: return FieldValue::UP(new StringFieldValue); - case T_STRING: return FieldValue::UP(new StringFieldValue); - case T_RAW: return FieldValue::UP(new RawFieldValue); - case T_LONG: return FieldValue::UP(new LongFieldValue); - case T_DOUBLE: return FieldValue::UP(new DoubleFieldValue); - case T_BYTE: return FieldValue::UP(new ByteFieldValue); - case T_PREDICATE: return FieldValue::UP(new PredicateFieldValue); + case T_INT: return std::make_unique<IntFieldValue>(); + case T_SHORT: return std::make_unique<ShortFieldValue>(); + case T_FLOAT: return std::make_unique<FloatFieldValue>(); + case T_URI: return std::make_unique<StringFieldValue>(); + case T_STRING: return std::make_unique<StringFieldValue>(); + case T_RAW: return std::make_unique<RawFieldValue>(); + case T_LONG: return std::make_unique<LongFieldValue>(); + case T_DOUBLE: return std::make_unique<DoubleFieldValue>(); + case T_BOOL: return std::make_unique<BoolFieldValue>(); + case T_BYTE: return std::make_unique<ByteFieldValue>(); + case T_PREDICATE: return std::make_unique<PredicateFieldValue>(); case T_TENSOR: return std::make_unique<TensorFieldValue>(); } LOG_ABORT("getId() returned value out of range"); diff --git a/document/src/vespa/document/fieldvalue/CMakeLists.txt b/document/src/vespa/document/fieldvalue/CMakeLists.txt index 0b161cff08a..dcef1bc8305 100644 --- a/document/src/vespa/document/fieldvalue/CMakeLists.txt +++ b/document/src/vespa/document/fieldvalue/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(document_fieldvalues OBJECT SOURCES annotationreferencefieldvalue.cpp arrayfieldvalue.cpp + boolfieldvalue.cpp bytefieldvalue.cpp collectionfieldvalue.cpp document.cpp diff --git a/document/src/vespa/document/fieldvalue/boolfieldvalue.cpp b/document/src/vespa/document/fieldvalue/boolfieldvalue.cpp new file mode 100644 index 00000000000..88ad1ddd11b --- /dev/null +++ b/document/src/vespa/document/fieldvalue/boolfieldvalue.cpp @@ -0,0 +1,85 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "boolfieldvalue.h" +#include <vespa/document/datatype/datatype.h> +#include <vespa/vespalib/util/xmlstream.h> +#include <ostream> + +using namespace vespalib::xml; + +namespace document { + +IMPLEMENT_IDENTIFIABLE(BoolFieldValue, FieldValue); + +BoolFieldValue::BoolFieldValue(bool value) + : _value(value), _altered(false) { +} + +BoolFieldValue::~BoolFieldValue() = default; + +FieldValue &BoolFieldValue::assign(const FieldValue &rhs) { + if (rhs.inherits(BoolFieldValue::classId)) { + operator=(static_cast<const BoolFieldValue &>(rhs)); + return *this; + } else { + _altered = true; + return FieldValue::assign(rhs); + } +} + +int BoolFieldValue::compare(const FieldValue&rhs) const { + int diff = FieldValue::compare(rhs); + if (diff != 0) return diff; + const BoolFieldValue &o = static_cast<const BoolFieldValue &>(rhs); + return (_value == o._value) ? 0 : _value ? 1 : -1; +} + +void BoolFieldValue::printXml(XmlOutputStream& out) const { + out << XmlContent(getAsString()); +} + +void BoolFieldValue::print(std::ostream& out, bool, const std::string&) const { + out << (_value ? "true" : "false") << "\n"; +} + +const DataType * +BoolFieldValue::getDataType() const { + return DataType::BOOL; +} + +bool +BoolFieldValue::hasChanged() const { + return _altered; +} + +FieldValue * +BoolFieldValue::clone() const { + return new BoolFieldValue(*this); +} + +char +BoolFieldValue::getAsByte() const { + return _value ? 1 : 0; +} +int32_t +BoolFieldValue::getAsInt() const { + return _value ? 1 : 0; +} +int64_t +BoolFieldValue::getAsLong() const { + return _value ? 1 : 0; +} +float +BoolFieldValue::getAsFloat() const { + return _value ? 1 : 0; +} +double +BoolFieldValue::getAsDouble() const { + return _value ? 1 : 0; +} +vespalib::string +BoolFieldValue::getAsString() const { + return _value ? "true" : "false"; +} + +} // namespace document diff --git a/document/src/vespa/document/fieldvalue/boolfieldvalue.h b/document/src/vespa/document/fieldvalue/boolfieldvalue.h new file mode 100644 index 00000000000..689bd3f4d53 --- /dev/null +++ b/document/src/vespa/document/fieldvalue/boolfieldvalue.h @@ -0,0 +1,47 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "fieldvalue.h" + +namespace document { + +/** + * Represent the value in a filed of type 'bool' which can be either true or false. + **/ +class BoolFieldValue : public FieldValue { + bool _value; + bool _altered; + +public: + BoolFieldValue(bool value=false); + ~BoolFieldValue() override; + + void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); } + void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); } + + FieldValue *clone() const override; + int compare(const FieldValue &rhs) const override; + + void printXml(XmlOutputStream &out) const override; + void print(std::ostream &out, bool verbose, const std::string &indent) const override; + + const DataType *getDataType() const override; + bool hasChanged() const override; + + bool getValue() const { return _value; } + void setValue(bool v) { _value = v; } + + FieldValue &assign(const FieldValue &rhs) override; + + char getAsByte() const override; + int32_t getAsInt() const override; + int64_t getAsLong() const override; + float getAsFloat() const override; + double getAsDouble() const override; + vespalib::string getAsString() const override; + + DECLARE_IDENTIFIABLE(BoolFieldValue); +}; + +} diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index 7acc7e97be9..d915d9fd66d 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/base/exceptions.h> +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/util/xmlstream.h> #include <sstream> diff --git a/document/src/vespa/document/fieldvalue/fieldvalues.h b/document/src/vespa/document/fieldvalue/fieldvalues.h index 6d438bcd0d5..cc665cee95c 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalues.h +++ b/document/src/vespa/document/fieldvalue/fieldvalues.h @@ -2,18 +2,19 @@ #pragma once -#include <vespa/document/fieldvalue/arrayfieldvalue.h> -#include <vespa/document/fieldvalue/bytefieldvalue.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/fieldvalue/doublefieldvalue.h> -#include <vespa/document/fieldvalue/floatfieldvalue.h> -#include <vespa/document/fieldvalue/intfieldvalue.h> -#include <vespa/document/fieldvalue/longfieldvalue.h> -#include <vespa/document/fieldvalue/mapfieldvalue.h> -#include <vespa/document/fieldvalue/predicatefieldvalue.h> -#include <vespa/document/fieldvalue/rawfieldvalue.h> -#include <vespa/document/fieldvalue/shortfieldvalue.h> -#include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/document/fieldvalue/weightedsetfieldvalue.h> -#include <vespa/document/fieldvalue/tensorfieldvalue.h> +#include "arrayfieldvalue.h" +#include "boolfieldvalue.h" +#include "bytefieldvalue.h" +#include "document.h" +#include "doublefieldvalue.h" +#include "floatfieldvalue.h" +#include "intfieldvalue.h" +#include "longfieldvalue.h" +#include "mapfieldvalue.h" +#include "predicatefieldvalue.h" +#include "rawfieldvalue.h" +#include "shortfieldvalue.h" +#include "stringfieldvalue.h" +#include "weightedsetfieldvalue.h" +#include "tensorfieldvalue.h" diff --git a/document/src/vespa/document/fieldvalue/fieldvaluevisitor.h b/document/src/vespa/document/fieldvalue/fieldvaluevisitor.h index ef07dd25212..778b1c77023 100644 --- a/document/src/vespa/document/fieldvalue/fieldvaluevisitor.h +++ b/document/src/vespa/document/fieldvalue/fieldvaluevisitor.h @@ -5,6 +5,7 @@ namespace document { class AnnotationReferenceFieldValue; class ArrayFieldValue; +class BoolFieldValue; class ByteFieldValue; class Document; class DoubleFieldValue; @@ -26,6 +27,7 @@ struct FieldValueVisitor { virtual void visit(AnnotationReferenceFieldValue &value) = 0; virtual void visit(ArrayFieldValue &value) = 0; + virtual void visit(BoolFieldValue &value) = 0; virtual void visit(ByteFieldValue &value) = 0; virtual void visit(Document &value) = 0; virtual void visit(DoubleFieldValue &value) = 0; @@ -48,6 +50,7 @@ struct ConstFieldValueVisitor { virtual void visit(const AnnotationReferenceFieldValue &value) = 0; virtual void visit(const ArrayFieldValue &value) = 0; + virtual void visit(const BoolFieldValue &value) = 0; virtual void visit(const ByteFieldValue &value) = 0; virtual void visit(const Document &value) = 0; virtual void visit(const DoubleFieldValue &value) = 0; diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp b/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp index 90c93e7a944..91873f021d1 100644 --- a/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp +++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp @@ -27,8 +27,7 @@ NumericFieldValue<Number>::assign(const FieldValue& value) _value = static_cast<Number>(value.getAsLong()); } else if (value.getClass().id() == IDENTIFIABLE_CLASSID(FloatFieldValue)) { _value = static_cast<Number>(value.getAsFloat()); - } else if (value.getClass().id() == IDENTIFIABLE_CLASSID(DoubleFieldValue)) - { + } else if (value.getClass().id() == IDENTIFIABLE_CLASSID(DoubleFieldValue)) { _value = static_cast<Number>(value.getAsDouble()); } else { return FieldValue::assign(value); diff --git a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h index d5c58e862f5..e0df3a38353 100644 --- a/document/src/vespa/document/fieldvalue/predicatefieldvalue.h +++ b/document/src/vespa/document/fieldvalue/predicatefieldvalue.h @@ -40,5 +40,4 @@ public: DECLARE_IDENTIFIABLE(PredicateFieldValue); }; -} // namespace document - +} diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index 4f30851ac4c..89172b0bc46 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -5,6 +5,7 @@ #include <vespa/document/annotation/spantree.h> #include <vespa/document/fieldvalue/annotationreferencefieldvalue.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/boolfieldvalue.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/doublefieldvalue.h> @@ -190,12 +191,12 @@ void VespaDocumentDeserializer::read(MapFieldValue &value) { } namespace { -template <typename T> struct ValueType { typedef typename T::Number Type; }; -template <> struct ValueType<ShortFieldValue> { typedef uint16_t Type; }; -template <> struct ValueType<IntFieldValue> { typedef uint32_t Type; }; -template <> struct ValueType<LongFieldValue> { typedef uint64_t Type; }; -template <> -struct ValueType<RawFieldValue> { typedef vespalib::string Type; }; +template <typename T> struct ValueType { using Type = typename T::Number; }; +template <> struct ValueType<BoolFieldValue> { using Type = bool; }; +template <> struct ValueType<ShortFieldValue> { using Type = uint16_t; }; +template <> struct ValueType<IntFieldValue> { using Type = uint32_t; }; +template <> struct ValueType<LongFieldValue> { using Type = uint64_t; }; +template <> struct ValueType<RawFieldValue> { using Type = vespalib::string; }; template <typename T> void readFieldValue(nbostream &input, T &value) { @@ -214,6 +215,10 @@ stringref readAttributeString(Input &input) { } } // namespace +void VespaDocumentDeserializer::read(BoolFieldValue &value) { + readFieldValue(_stream, value); +} + void VespaDocumentDeserializer::read(ByteFieldValue &value) { readFieldValue(_stream, value); } diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.h b/document/src/vespa/document/serialization/vespadocumentdeserializer.h index 64346428cdd..e6b490e1075 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.h @@ -21,6 +21,7 @@ class VespaDocumentDeserializer : private FieldValueVisitor { void visit(AnnotationReferenceFieldValue &value) override { read(value); } void visit(ArrayFieldValue &value) override { read(value); } + void visit(BoolFieldValue &value) override { read(value); } void visit(ByteFieldValue &value) override { read(value); } void visit(Document &value) override { read(value); } void visit(DoubleFieldValue &value) override { read(value); } @@ -63,6 +64,7 @@ public: void read(AnnotationReferenceFieldValue &value); void read(ArrayFieldValue &value); void read(MapFieldValue &value); + void read(BoolFieldValue &value); void read(ByteFieldValue &value); void read(DoubleFieldValue &value); void read(FloatFieldValue &value); diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index ae03bcc0d3c..3519f38baab 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -6,6 +6,7 @@ #include "util.h" #include <vespa/document/fieldvalue/annotationreferencefieldvalue.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/boolfieldvalue.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/doublefieldvalue.h> @@ -24,6 +25,7 @@ #include <vespa/document/update/updates.h> #include <vespa/document/update/fieldpathupdates.h> #include <vespa/document/util/bytebuffer.h> +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/databuffer.h> @@ -185,6 +187,10 @@ void VespaDocumentSerializer::write(const MapFieldValue &value) { } } +void VespaDocumentSerializer::write(const BoolFieldValue &value) { + _stream << value.getValue(); +} + void VespaDocumentSerializer::write(const ByteFieldValue &value) { _stream << value.getValue(); } diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.h b/document/src/vespa/document/serialization/vespadocumentserializer.h index 818759d35b5..fde073c13b3 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentserializer.h @@ -4,12 +4,10 @@ #include <vespa/document/fieldvalue/fieldvaluevisitor.h> #include <vespa/document/fieldvalue/fieldvaluewriter.h> -#include <vespa/document/fieldset/fieldsets.h> +#include <vespa/document/fieldset/fieldset.h> #include <vespa/document/update/updatevisitor.h> -namespace vespalib { - class nbostream; -} +namespace vespalib { class nbostream; } namespace document { @@ -40,6 +38,7 @@ public: void write(const AnnotationReferenceFieldValue &value); void write(const ArrayFieldValue &value); void write(const MapFieldValue &map); + void write(const BoolFieldValue &value); void write(const ByteFieldValue &value); void write(const DoubleFieldValue &val); void write(const FloatFieldValue &value); @@ -90,6 +89,7 @@ private: void visit(const AnnotationReferenceFieldValue &value) override { write(value); } void visit(const ArrayFieldValue &value) override { write(value); } + void visit(const BoolFieldValue &value) override { write(value); } void visit(const ByteFieldValue &value) override { write(value); } void visit(const Document &value) override { write(value, COMPLETE); } void visit(const DoubleFieldValue &value) override { write(value); } diff --git a/document/src/vespa/document/util/identifiableid.h b/document/src/vespa/document/util/identifiableid.h index c52888e8491..84cfc506bcc 100644 --- a/document/src/vespa/document/util/identifiableid.h +++ b/document/src/vespa/document/util/identifiableid.h @@ -19,7 +19,7 @@ #define CID_StringFieldValue DOCUMENT_CID(15) #define CID_RawFieldValue DOCUMENT_CID(16) //Gone with vespa 6 #define CID_ContentFieldValue DOCUMENT_CID(17) -//Long gone #define CID_ContentMetaFieldValue DOCUMENT_CID(18) +#define CID_BoolFieldValue DOCUMENT_CID(18) #define CID_ArrayFieldValue DOCUMENT_CID(19) #define CID_WeightedSetFieldValue DOCUMENT_CID(20) #define CID_FieldMapValue DOCUMENT_CID(21) diff --git a/flags/pom.xml b/flags/pom.xml index 99fde3faebb..fc38676ff20 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -38,6 +38,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> <scope>provided</scope> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index 581ec599aab..7d84efa52b2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -17,8 +17,7 @@ import java.util.function.Consumer; * @author hakonhall */ @Immutable -public -class FetchVector { +public class FetchVector { public enum Dimension { /** Value from ZoneId::value */ ZONE_ID, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java index f4e23144449..1a31ecd713e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FileFlagSource.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.flags.json.Rule; @@ -21,8 +20,6 @@ import java.util.Optional; * @author hakonhall */ public class FileFlagSource implements FlagSource { - private static final ObjectMapper mapper = new ObjectMapper(); - static final String FLAGS_DIRECTORY = "/etc/vespa/flags"; private final Path flagsDirectory; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java index da8e6b29cab..182ab85858c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagSource.java @@ -4,8 +4,11 @@ package com.yahoo.vespa.flags; import java.util.Optional; /** + * A source of raw flag values that can be converted to typed flag values elsewhere. + * * @author hakonhall */ public interface FlagSource { + /** Get raw flag for the given vector (specifying hostname, application id, etc). */ Optional<RawFlag> fetch(FlagId id, FetchVector vector); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 6773c02e441..545f288af29 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -30,7 +30,7 @@ public class Flags { HOSTNAME); public static final UnboundBooleanFlag DUPERMODEL_USE_CONFIGSERVERCONFIG = defineFeatureFlag( - "dupermodel-use-configserverconfig", true, + "dupermodel-use-configserverconfig", false, "For historical reasons, the ApplicationInfo in the DuperModel for controllers and config servers " + "is based on the ConfigserverConfig (this flag is true). We want to transition to use the " + "infrastructure application activated by the InfrastructureProvisioner once that supports health.", @@ -71,6 +71,12 @@ public class Flags { "Whether to enable Nessus.", "Takes effect on next host admin tick", HOSTNAME); + public static final UnboundBooleanFlag ENABLE_CPU_TEMPERATURE_TASK = defineFeatureFlag( + "enable-cputemptask", true, + "Whether to enable CPU temperature task", "Takes effect on next host admin tick", + HOSTNAME + ); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java new file mode 100644 index 00000000000..abe8d407ab0 --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java @@ -0,0 +1,111 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.file; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Java API for a flag database stored in a single file + * + * @author hakonhall + */ +public class FlagDbFile { + private static final Logger logger = Logger.getLogger(FlagDbFile.class.getName()); + + private final Path path; + + public FlagDbFile() { + this(FileSystems.getDefault()); + } + + public FlagDbFile(FileSystem fileSystem) { + this(fileSystem.getPath(Defaults.getDefaults().vespaHome() + "/var/vespa/flag.db")); + } + + public FlagDbFile(Path path) { + this.path = path; + } + + public Path getPath() { + return path; + } + + public Map<FlagId, FlagData> read() { + Optional<byte[]> bytes = readFile(); + if (!bytes.isPresent()) return Collections.emptyMap(); + return FlagData.deserializeList(bytes.get()).stream().collect(Collectors.toMap(FlagData::id, Function.identity())); + } + + public boolean sync(Map<FlagId, FlagData> flagData) { + boolean modified = false; + Map<FlagId, FlagData> currentFlagData = read(); + Set<FlagId> flagIdsToBeRemoved = new HashSet<>(currentFlagData.keySet()); + List<FlagData> flagDataList = new ArrayList<>(flagData.values()); + + for (FlagData data : flagDataList) { + flagIdsToBeRemoved.remove(data.id()); + + FlagData existingFlagData = currentFlagData.get(data.id()); + if (existingFlagData == null) { + logger.log(LogLevel.INFO, "New flag " + data.id() + ": " + data.serializeToJson()); + modified = true; + + // Could also consider testing with FlagData::equals, but that would be too fragile? + } else if (!Objects.equals(data.serializeToJson(), existingFlagData.serializeToJson())){ + logger.log(LogLevel.INFO, "Updating flag " + data.id() + " from " + + existingFlagData.serializeToJson() + " to " + data.serializeToJson()); + modified = true; + } + } + + if (!flagIdsToBeRemoved.isEmpty()) { + String flagIdsString = flagIdsToBeRemoved.stream().map(FlagId::toString).collect(Collectors.joining(", ")); + logger.log(LogLevel.INFO, "Removing flags " + flagIdsString); + modified = true; + } + + if (!modified) return false; + + writeFile(FlagData.serializeListToUtf8Json(flagDataList)); + + return modified; + } + + private Optional<byte[]> readFile() { + try { + return Optional.of(Files.readAllBytes(path)); + } catch (NoSuchFileException e) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void writeFile(byte[] bytes) { + uncheck(() -> Files.createDirectories(path.getParent())); + uncheck(() -> Files.write(path, bytes)); + } +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java new file mode 100644 index 00000000000..27ad44f938e --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.flags.file; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java index 572ec511607..64c4bbe7616 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.RawFlag; import com.yahoo.vespa.flags.json.wire.WireFlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagDataList; import com.yahoo.vespa.flags.json.wire.WireRule; import javax.annotation.concurrent.Immutable; @@ -114,6 +115,24 @@ public class FlagData { ); } + public static byte[] serializeListToUtf8Json(List<FlagData> list) { + return listToWire(list).serializeToBytes(); + } + + public static List<FlagData> deserializeList(byte[] bytes) { + return listFromWire(WireFlagDataList.deserializeFrom(bytes)); + } + + public static WireFlagDataList listToWire(List<FlagData> list) { + WireFlagDataList wireList = new WireFlagDataList(); + wireList.flags = list.stream().map(FlagData::toWire).collect(Collectors.toList()); + return wireList; + } + + public static List<FlagData> listFromWire(WireFlagDataList wireList) { + return wireList.flags.stream().map(FlagData::fromWire).collect(Collectors.toList()); + } + private static List<Rule> rulesFromWire(List<WireRule> wireRules) { if (wireRules == null) return Collections.emptyList(); return wireRules.stream().map(Rule::fromWire).collect(Collectors.toList()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java new file mode 100644 index 00000000000..60b35d9b69e --- /dev/null +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/wire/WireFlagDataList.java @@ -0,0 +1,37 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.json.wire; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author hakonhall + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireFlagDataList { + @JsonProperty("flags") + public List<WireFlagData> flags = new ArrayList<>(); + + private static final ObjectMapper mapper = new ObjectMapper(); + + public void serializeToOutputStream(OutputStream outputStream) { + uncheck(() -> mapper.writeValue(outputStream, this)); + } + + public byte[] serializeToBytes() { + return uncheck(() -> mapper.writeValueAsBytes(this)); + } + + public static WireFlagDataList deserializeFrom(byte[] bytes) { + return uncheck(() -> mapper.readValue(bytes, WireFlagDataList.class)); + } +} diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java new file mode 100644 index 00000000000..fd1a71e4b4a --- /dev/null +++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java @@ -0,0 +1,76 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags.file; + +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.hamcrest.collection.IsMapContaining; +import org.hamcrest.collection.IsMapWithSize; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + * @author hakonhall + */ +public class FlagDbFileTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final FlagDbFile flagDb = new FlagDbFile(fileSystem); + + @Test + public void test() { + Map<FlagId, FlagData> dataMap = new HashMap<>(); + FlagId id1 = new FlagId("id1"); + FlagData data1 = new FlagData(id1, new FetchVector()); + dataMap.put(id1, data1); + FlagId id2 = new FlagId("id2"); + FlagData data2 = new FlagData(id2, new FetchVector()); + dataMap.put(id2, data2); + + // Non-existing directory => empty map + assertThat(flagDb.read(), IsMapWithSize.anEmptyMap()); + + // sync() will create directory with map content + assertThat(flagDb.sync(dataMap), equalTo(true)); + Map<FlagId, FlagData> readDataMap = flagDb.read(); + assertThat(readDataMap, IsMapWithSize.aMapWithSize(2)); + assertThat(readDataMap, IsMapContaining.hasKey(id1)); + assertThat(readDataMap, IsMapContaining.hasKey(id2)); + + assertThat(getDbContent(), equalTo("{\"flags\":[{\"id\":\"id1\"},{\"id\":\"id2\"}]}")); + + // another sync with the same data is a no-op + assertThat(flagDb.sync(dataMap), equalTo(false)); + + // Changing value of id1, removing id2, adding id3 + dataMap.remove(id2); + FlagData newData1 = new FlagData(id1, new FetchVector().with(FetchVector.Dimension.HOSTNAME, "h1")); + dataMap.put(id1, newData1); + FlagId id3 = new FlagId("id3"); + FlagData data3 = new FlagData(id3, new FetchVector()); + dataMap.put(id3, data3); + assertThat(flagDb.sync(dataMap), equalTo(true)); + Map<FlagId, FlagData> anotherReadDataMap = flagDb.read(); + assertThat(anotherReadDataMap, IsMapWithSize.aMapWithSize(2)); + assertThat(anotherReadDataMap, IsMapContaining.hasKey(id1)); + assertThat(anotherReadDataMap, IsMapContaining.hasKey(id3)); + assertThat(anotherReadDataMap.get(id1).serializeToJson(), equalTo("{\"id\":\"id1\",\"attributes\":{\"hostname\":\"h1\"}}")); + + assertThat(flagDb.sync(Collections.emptyMap()), equalTo(true)); + assertThat(getDbContent(), equalTo("{\"flags\":[]}")); + } + + public String getDbContent() { + return uncheck(() -> new String(Files.readAllBytes(flagDb.getPath()), StandardCharsets.UTF_8)); + } +}
\ No newline at end of file diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java index c964dfce2c7..f51026382fc 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/RetryingJaxRsStrategy.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.jaxrs.client; import com.yahoo.vespa.applicationmodel.HostName; import javax.ws.rs.ProcessingException; +import javax.ws.rs.ServiceUnavailableException; import javax.ws.rs.core.UriBuilder; import java.io.IOException; import java.net.URI; @@ -73,7 +74,7 @@ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { @Override public <R> R apply(final Function<T, R> function, JaxRsTimeouts timeouts) throws IOException { - ProcessingException sampleException = null; + RuntimeException sampleException = null; for (int i = 0; i < maxIterations; ++i) { for (final HostName hostName : hostNames) { @@ -84,8 +85,9 @@ public class RetryingJaxRsStrategy<T> implements JaxRsStrategy<T> { final T jaxRsClient = jaxRsClientFactory.createClient(params); try { return function.apply(jaxRsClient); - } catch (ProcessingException e) { - // E.g. java.net.SocketTimeoutException thrown on read timeout is wrapped as a ProcessingException + } catch (ProcessingException | ServiceUnavailableException e) { + // E.g. java.net.SocketTimeoutException thrown on read timeout is wrapped as a ProcessingException, + // while ServiceUnavailableException is a WebApplicationException sampleException = e; logger.log(Level.INFO, "Failed REST API call to " + hostName + ":" + port + pathPrefix + " (in retry loop): " diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java index 65aafee0060..2a1967abd13 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java @@ -389,7 +389,7 @@ public class Request extends AbstractResource { * @throws BindingNotFoundException If the corresponding call to {@link Container#resolveHandler(Request)} returns * null. */ - public ContentChannel connect(final ResponseHandler responseHandler) { + public ContentChannel connect(ResponseHandler responseHandler) { try { Objects.requireNonNull(responseHandler, "responseHandler"); RequestHandler requestHandler = container().resolveHandler(this); diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 476902e400c..e1231f2585d 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -49,6 +49,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>flags</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <!-- Compile --> <dependency> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java index 4a2496f4d3e..ab899f9f919 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import com.yahoo.vespa.hosted.node.admin.configserver.flags.FlagRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.state.State; @@ -18,7 +19,10 @@ public interface ConfigServerClients { Orchestrator orchestrator(); /** Get handle to the /state/v1 REST API */ - default State state() { throw new UnsupportedOperationException(); } + State state(); + + /** Get handle to the /flags/v1 REST API */ + FlagRepository flagRepository(); void stop(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java index 6c982bfa71c..af11c300c2b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import com.yahoo.vespa.hosted.node.admin.configserver.flags.FlagRepository; +import com.yahoo.vespa.hosted.node.admin.configserver.flags.RealFlagRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; @@ -19,6 +21,7 @@ public class RealConfigServerClients implements ConfigServerClients { private final NodeRepository nodeRepository; private final Orchestrator orchestrator; private final State state; + private final RealFlagRepository flagRepository; /** * @param configServerApi the backend API to use - will be closed at {@link #stop()}. @@ -28,6 +31,7 @@ public class RealConfigServerClients implements ConfigServerClients { nodeRepository = new RealNodeRepository(configServerApi); orchestrator = new OrchestratorImpl(configServerApi); state = new StateImpl(configServerApi); + flagRepository = new RealFlagRepository(configServerApi); } @Override @@ -46,6 +50,11 @@ public class RealConfigServerClients implements ConfigServerClients { } @Override + public FlagRepository flagRepository() { + return flagRepository; + } + + @Override public void stop() { configServerApi.close(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/FlagRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/FlagRepository.java new file mode 100644 index 00000000000..8407d42131b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/FlagRepository.java @@ -0,0 +1,15 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.flags; + +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; + +import java.util.List; +import java.util.Map; + +/** + * @author hakonhall + */ +public interface FlagRepository { + Map<FlagId, FlagData> getAllFlagData(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java new file mode 100644 index 00000000000..a017569294e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepository.java @@ -0,0 +1,28 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.flags; + +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagDataList; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; + +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * @author hakonhall + */ +public class RealFlagRepository implements FlagRepository { + private final ConfigServerApi configServerApi; + + public RealFlagRepository(ConfigServerApi configServerApi) { + this.configServerApi = configServerApi; + } + + @Override + public Map<FlagId, FlagData> getAllFlagData() { + WireFlagDataList list = configServerApi.get("/flags/v1/data?recursive=true", WireFlagDataList.class); + return FlagData.listFromWire(list).stream().collect(Collectors.toMap(FlagData::id, Function.identity())); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java new file mode 100644 index 00000000000..b991adfc639 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.configserver.flags; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java index c65d59a79dc..af8dfb1fd27 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; import java.util.List; @@ -15,7 +14,7 @@ import java.util.Optional; public interface DockerOperations { - void createContainer(NodeAgentContext context, NodeSpec node, ContainerData containerData); + void createContainer(NodeAgentContext context, ContainerData containerData); void startContainer(NodeAgentContext context); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 89ab2e60b63..e1b77b6a41b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; @@ -55,19 +54,19 @@ public class DockerOperationsImpl implements DockerOperations { } @Override - public void createContainer(NodeAgentContext context, NodeSpec node, ContainerData containerData) { + public void createContainer(NodeAgentContext context, ContainerData containerData) { context.log(logger, "Creating container"); // IPv6 - Assume always valid - Inet6Address ipV6Address = ipAddresses.getIPv6Address(node.getHostname()).orElseThrow( - () -> new RuntimeException("Unable to find a valid IPv6 address for " + node.getHostname() + + Inet6Address ipV6Address = ipAddresses.getIPv6Address(context.node().getHostname()).orElseThrow( + () -> new RuntimeException("Unable to find a valid IPv6 address for " + context.node().getHostname() + ". Missing an AAAA DNS entry?")); Docker.CreateContainerCommand command = docker.createContainerCommand( - node.getWantedDockerImage().get(), - ContainerResources.from(node.getMinCpuCores(), node.getMinMainMemoryAvailableGb()), + context.node().getWantedDockerImage().get(), + ContainerResources.from(context.node().getMinCpuCores(), context.node().getMinMainMemoryAvailableGb()), context.containerName(), - node.getHostname()) + context.node().getHostname()) .withManagedBy(MANAGER_NAME) .withUlimit("nofile", 262_144, 262_144) // The nproc aka RLIMIT_NPROC resource limit works as follows: @@ -100,20 +99,20 @@ public class DockerOperationsImpl implements DockerOperations { command.withIpAddress(ipV6Local); // IPv4 - Only present for some containers - Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(node.getHostname()) + Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().getHostname()) .map(ipV4Address -> { InetAddress ipV4Prefix = InetAddresses.forString(IPV4_NPT_PREFIX); return IPAddresses.prefixTranslate(ipV4Address, ipV4Prefix, 2); }); ipV4Local.ifPresent(command::withIpAddress); - addEtcHosts(containerData, node.getHostname(), ipV4Local, ipV6Local); + addEtcHosts(containerData, context.node().getHostname(), ipV4Local, ipV6Local); } addMounts(context, command); // TODO: Enforce disk constraints - long minMainMemoryAvailableMb = (long) (node.getMinMainMemoryAvailableGb() * 1024); + long minMainMemoryAvailableMb = (long) (context.node().getMinMainMemoryAvailableGb() * 1024); if (minMainMemoryAvailableMb > 0) { // VESPA_TOTAL_MEMORY_MB is used to make any jdisc container think the machine // only has this much physical memory (overrides total memory reported by `free -m`). diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 2fd40a1b486..47255c54455 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; @@ -65,9 +64,9 @@ public class StorageMaintainer { this.archiveContainerStoragePath = archiveContainerStoragePath; } - public void writeMetricsConfig(NodeAgentContext context, NodeSpec node) { + public void writeMetricsConfig(NodeAgentContext context) { List<SecretAgentCheckConfig> configs = new ArrayList<>(); - Map<String, Object> tags = generateTags(context, node); + Map<String, Object> tags = generateTags(context); // host-life Path hostLifeCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life"); @@ -154,26 +153,26 @@ public class StorageMaintainer { dockerOperations.executeCommandInContainerAsRoot(context, "service", "yamas-agent", "restart"); } - private Map<String, Object> generateTags(NodeAgentContext context, NodeSpec node) { + private Map<String, Object> generateTags(NodeAgentContext context) { Map<String, String> tags = new LinkedHashMap<>(); tags.put("namespace", "Vespa"); - tags.put("role", nodeTypeToRole(node.getNodeType())); + tags.put("role", nodeTypeToRole(context.node().getNodeType())); tags.put("zone", String.format("%s.%s", context.zoneId().environment().value(), context.zoneId().regionName().value())); - node.getVespaVersion().ifPresent(version -> tags.put("vespaVersion", version)); + context.node().getVespaVersion().ifPresent(version -> tags.put("vespaVersion", version)); if (! isConfigserverLike(context.nodeType())) { - tags.put("flavor", node.getFlavor()); - tags.put("canonicalFlavor", node.getCanonicalFlavor()); - tags.put("state", node.getState().toString()); - node.getParentHostname().ifPresent(parent -> tags.put("parentHostname", parent)); - node.getOwner().ifPresent(owner -> { + tags.put("flavor", context.node().getFlavor()); + tags.put("canonicalFlavor", context.node().getCanonicalFlavor()); + tags.put("state", context.node().getState().toString()); + context.node().getParentHostname().ifPresent(parent -> tags.put("parentHostname", parent)); + context.node().getOwner().ifPresent(owner -> { tags.put("tenantName", owner.getTenant()); tags.put("app", owner.getApplication() + "." + owner.getInstance()); tags.put("applicationName", owner.getApplication()); tags.put("instanceName", owner.getInstance()); tags.put("applicationId", owner.getTenant() + "." + owner.getApplication() + "." + owner.getInstance()); }); - node.getMembership().ifPresent(membership -> { + context.node().getMembership().ifPresent(membership -> { tags.put("clustertype", membership.getClusterType()); tags.put("clusterid", membership.getClusterId()); }); @@ -250,26 +249,30 @@ public class StorageMaintainer { FileFinder.directories(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/db/vespa/filedistribution"))) .match(olderThan(Duration.ofDays(31))) .deleteRecursively(); + + FileFinder.directories(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/db/vespa/download"))) + .match(olderThan(Duration.ofDays(31))) + .deleteRecursively(); } /** Checks if container has any new coredumps, reports and archives them if so */ - public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node, Optional<Container> container) { - final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(context, node, container); + public void handleCoreDumpsForContainer(NodeAgentContext context, Optional<Container> container) { + final Map<String, Object> nodeAttributes = getCoredumpNodeAttributes(context, container); coredumpHandler.converge(context, nodeAttributes); } - private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, NodeSpec node, Optional<Container> container) { + private Map<String, Object> getCoredumpNodeAttributes(NodeAgentContext context, Optional<Container> container) { Map<String, String> attributes = new HashMap<>(); - attributes.put("hostname", node.getHostname()); + attributes.put("hostname", context.node().getHostname()); attributes.put("region", context.zoneId().regionName().value()); attributes.put("environment", context.zoneId().environment().value()); - attributes.put("flavor", node.getFlavor()); + attributes.put("flavor", context.node().getFlavor()); attributes.put("kernel_version", System.getProperty("os.version")); container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString())); - node.getParentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); - node.getVespaVersion().ifPresent(version -> attributes.put("vespa_version", version)); - node.getOwner().ifPresent(owner -> { + context.node().getParentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); + context.node().getVespaVersion().ifPresent(version -> attributes.put("vespa_version", version)); + context.node().getOwner().ifPresent(owner -> { attributes.put("tenant", owner.getTenant()); attributes.put("application", owner.getApplication()); attributes.put("instance", owner.getInstance()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index b191401b8e0..2303f78217c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -9,6 +9,11 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextManager; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentScheduler; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import java.time.Clock; @@ -23,7 +28,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import java.util.stream.Collectors; /** @@ -33,12 +37,15 @@ import java.util.stream.Collectors; */ public class NodeAdminImpl implements NodeAdmin { private static final PrefixLogger logger = PrefixLogger.getNodeAdminLogger(NodeAdmin.class); + private static final Duration NODE_AGENT_FREEZE_TIMEOUT = Duration.ofSeconds(5); + private final ScheduledExecutorService aclScheduler = Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("aclscheduler")); private final ScheduledExecutorService metricsScheduler = Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("metricsscheduler")); - private final Function<String, NodeAgent> nodeAgentFactory; + private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory; + private final NodeAgentContextFactory nodeAgentContextFactory; private final Optional<AclMaintainer> aclMaintainer; private final Clock clock; @@ -46,16 +53,27 @@ public class NodeAdminImpl implements NodeAdmin { private boolean isFrozen; private Instant startOfFreezeConvergence; - private final Map<String, NodeAgent> nodeAgentsByHostname = new ConcurrentHashMap<>(); + private final Map<String, NodeAgentWithScheduler> nodeAgentWithSchedulerByHostname = new ConcurrentHashMap<>(); private final GaugeWrapper numberOfContainersInLoadImageState; private final CounterWrapper numberOfUnhandledExceptionsInNodeAgent; - public NodeAdminImpl(Function<String, NodeAgent> nodeAgentFactory, + public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, + NodeAgentContextFactory nodeAgentContextFactory, Optional<AclMaintainer> aclMaintainer, MetricReceiverWrapper metricReceiver, Clock clock) { - this.nodeAgentFactory = nodeAgentFactory; + this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), + nodeAgentContextFactory, aclMaintainer, metricReceiver, clock); + } + + NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory, + NodeAgentContextFactory nodeAgentContextFactory, + Optional<AclMaintainer> aclMaintainer, + MetricReceiverWrapper metricReceiver, + Clock clock) { + this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory; + this.nodeAgentContextFactory = nodeAgentContextFactory; this.aclMaintainer = aclMaintainer; this.clock = clock; @@ -70,22 +88,33 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void refreshContainersToRun(List<NodeSpec> containersToRun) { - final Set<String> hostnamesOfContainersToRun = containersToRun.stream() - .map(NodeSpec::getHostname) - .collect(Collectors.toSet()); + final Map<String, NodeAgentContext> nodeAgentContextsByHostname = containersToRun.stream() + .collect(Collectors.toMap(NodeSpec::getHostname, nodeAgentContextFactory::create)); - synchronizeNodesToNodeAgents(hostnamesOfContainersToRun); + // Stop and remove NodeAgents that should no longer be running + diff(nodeAgentWithSchedulerByHostname.keySet(), nodeAgentContextsByHostname.keySet()) + .forEach(hostname -> nodeAgentWithSchedulerByHostname.remove(hostname).stop()); + + // Start NodeAgent for hostnames that should be running, but aren't yet + diff(nodeAgentContextsByHostname.keySet(), nodeAgentWithSchedulerByHostname.keySet()).forEach(hostname -> { + NodeAgentWithScheduler naws = nodeAgentWithSchedulerFactory.create(nodeAgentContextsByHostname.get(hostname)); + naws.start(); + nodeAgentWithSchedulerByHostname.put(hostname, naws); + }); - updateNodeAgentMetrics(); + // At this point, nodeAgentContextsByHostname and nodeAgentWithSchedulerByHostname should have the same keys + nodeAgentContextsByHostname.forEach((hostname, context) -> + nodeAgentWithSchedulerByHostname.get(hostname).scheduleTickWith(context) + ); } private void updateNodeAgentMetrics() { int numberContainersWaitingImage = 0; int numberOfNewUnhandledExceptions = 0; - for (NodeAgent nodeAgent : nodeAgentsByHostname.values()) { - if (nodeAgent.isDownloadingImage()) numberContainersWaitingImage++; - numberOfNewUnhandledExceptions += nodeAgent.getAndResetNumberOfUnhandledExceptions(); + for (NodeAgentWithScheduler nodeAgentWithScheduler : nodeAgentWithSchedulerByHostname.values()) { + if (nodeAgentWithScheduler.isDownloadingImage()) numberContainersWaitingImage++; + numberOfNewUnhandledExceptions += nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions(); } numberOfContainersInLoadImageState.sample(numberContainersWaitingImage); @@ -105,8 +134,8 @@ public class NodeAdminImpl implements NodeAdmin { } // Use filter with count instead of allMatch() because allMatch() will short circuit on first non-match - boolean allNodeAgentsConverged = nodeAgentsByHostname.values().stream() - .filter(nodeAgent -> !nodeAgent.setFrozen(wantFrozen)) + boolean allNodeAgentsConverged = nodeAgentWithSchedulerByHostname.values().parallelStream() + .filter(nodeAgentScheduler -> !nodeAgentScheduler.setFrozen(wantFrozen, NODE_AGENT_FREEZE_TIMEOUT)) .count() == 0; if (wantFrozen) { @@ -134,8 +163,8 @@ public class NodeAdminImpl implements NodeAdmin { public void stopNodeAgentServices(List<String> hostnames) { // Each container may spend 1-1:30 minutes stopping hostnames.parallelStream() - .filter(nodeAgentsByHostname::containsKey) - .map(nodeAgentsByHostname::get) + .filter(nodeAgentWithSchedulerByHostname::containsKey) + .map(nodeAgentWithSchedulerByHostname::get) .forEach(nodeAgent -> { nodeAgent.suspend(); nodeAgent.stopServices(); @@ -146,7 +175,8 @@ public class NodeAdminImpl implements NodeAdmin { public void start() { metricsScheduler.scheduleAtFixedRate(() -> { try { - nodeAgentsByHostname.values().forEach(NodeAgent::updateContainerNodeMetrics); + updateNodeAgentMetrics(); + nodeAgentWithSchedulerByHostname.values().forEach(NodeAgent::updateContainerNodeMetrics); } catch (Throwable e) { logger.warning("Metric fetcher scheduler failed", e); } @@ -166,7 +196,7 @@ public class NodeAdminImpl implements NodeAdmin { aclScheduler.shutdown(); // Stop all node-agents in parallel, will block until the last NodeAgent is stopped - nodeAgentsByHostname.values().parallelStream().forEach(NodeAgent::stop); + nodeAgentWithSchedulerByHostname.values().parallelStream().forEach(NodeAgent::stop); do { try { @@ -185,23 +215,35 @@ public class NodeAdminImpl implements NodeAdmin { return result; } - void synchronizeNodesToNodeAgents(Set<String> hostnamesToRun) { - // Stop and remove NodeAgents that should no longer be running - diff(nodeAgentsByHostname.keySet(), hostnamesToRun) - .forEach(hostname -> nodeAgentsByHostname.remove(hostname).stop()); + static class NodeAgentWithScheduler implements NodeAgent, NodeAgentScheduler { + private final NodeAgent nodeAgent; + private final NodeAgentScheduler nodeAgentScheduler; - // Start NodeAgent for hostnames that should be running, but aren't yet - diff(hostnamesToRun, nodeAgentsByHostname.keySet()) - .forEach(this::startNodeAgent); + private NodeAgentWithScheduler(NodeAgent nodeAgent, NodeAgentScheduler nodeAgentScheduler) { + this.nodeAgent = nodeAgent; + this.nodeAgentScheduler = nodeAgentScheduler; + } + + @Override public void stopServices() { nodeAgent.stopServices(); } + @Override public void suspend() { nodeAgent.suspend(); } + @Override public void start() { nodeAgent.start(); } + @Override public void stop() { nodeAgent.stop(); } + @Override public void updateContainerNodeMetrics() { nodeAgent.updateContainerNodeMetrics(); } + @Override public boolean isDownloadingImage() { return nodeAgent.isDownloadingImage(); } + @Override public int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); } + + @Override public void scheduleTickWith(NodeAgentContext context) { nodeAgentScheduler.scheduleTickWith(context); } + @Override public boolean setFrozen(boolean frozen, Duration timeout) { return nodeAgentScheduler.setFrozen(frozen, timeout); } } - private void startNodeAgent(String hostname) { - if (nodeAgentsByHostname.containsKey(hostname)) - throw new IllegalArgumentException("Attempted to start NodeAgent for hostname " + hostname + - ", but one is already running!"); + @FunctionalInterface + interface NodeAgentWithSchedulerFactory { + NodeAgentWithScheduler create(NodeAgentContext context); + } - NodeAgent agent = nodeAgentFactory.apply(hostname); - agent.start(); - nodeAgentsByHostname.put(hostname, agent); + private static NodeAgentWithScheduler create(Clock clock, NodeAgentFactory nodeAgentFactory, NodeAgentContext context) { + NodeAgentContextManager contextManager = new NodeAgentContextManager(clock, context); + NodeAgent nodeAgent = nodeAgentFactory.create(contextManager); + return new NodeAgentWithScheduler(nodeAgent, contextManager); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index a12104c6e98..13d3f3307d2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -51,30 +51,21 @@ public class NodeAdminStateUpdater { nodeAdmin.start(); } - public void converge(State wantedState) { - try { - convergeState(wantedState); - } finally { - if (wantedState != RESUMED && currentState == TRANSITIONING) { - Duration subsystemFreezeDuration = nodeAdmin.subsystemFreezeDuration(); - if (subsystemFreezeDuration.compareTo(FREEZE_CONVERGENCE_TIMEOUT) > 0) { - // We have spent too much time trying to freeze and node admin is still not frozen. - // To avoid node agents stalling for too long, we'll force unfrozen ticks now. - log.info("Timed out trying to freeze, will force unfreezed ticks"); - fetchContainersToRunFromNodeRepository(); - nodeAdmin.setFrozen(false); - } - } else if (currentState == RESUMED) { - fetchContainersToRunFromNodeRepository(); - } - } - } - /** * This method attempts to converge node-admin w/agents to a {@link State} * with respect to: freeze, Orchestrator, and services running. */ - private void convergeState(State wantedState) { + public void converge(State wantedState) { + if (wantedState == RESUMED) { + adjustNodeAgentsToRunFromNodeRepository(); + } else if (currentState == TRANSITIONING && nodeAdmin.subsystemFreezeDuration().compareTo(FREEZE_CONVERGENCE_TIMEOUT) > 0) { + // We have spent too much time trying to freeze and node admin is still not frozen. + // To avoid node agents stalling for too long, we'll force unfrozen ticks now. + adjustNodeAgentsToRunFromNodeRepository(); + nodeAdmin.setFrozen(false); + throw new ConvergenceException("Timed out trying to freeze all nodes: will force an unfrozen tick"); + } + if (currentState == wantedState) return; currentState = TRANSITIONING; @@ -119,7 +110,7 @@ public class NodeAdminStateUpdater { currentState = wantedState; } - private void fetchContainersToRunFromNodeRepository() { + private void adjustNodeAgentsToRunFromNodeRepository() { try { final List<NodeSpec> containersToRun = nodeRepository.getNodes(hostHostname); nodeAdmin.refreshContainersToRun(containersToRun); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java index 947e7c85d66..10076c4f48a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java @@ -9,12 +9,6 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; * @author bakksjo */ public interface NodeAgent { - /** - * Will eventually freeze/unfreeze the node agent - * @param frozen whether node agent should be frozen - * @return True if node agent has converged to the desired state - */ - boolean setFrozen(boolean frozen); /** * Stop services running on node. Depending on the state of the node, {@link #suspend()} might need to be diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index 4874eccb913..2e9f58a2c31 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.component.ZoneId; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; import java.nio.file.Path; @@ -13,11 +14,17 @@ import java.nio.file.Paths; public interface NodeAgentContext extends TaskContext { + NodeSpec node(); + ContainerName containerName(); - HostName hostname(); + default HostName hostname() { + return HostName.from(node().getHostname()); + } - NodeType nodeType(); + default NodeType nodeType() { + return node().getNodeType(); + } AthenzService identity(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java new file mode 100644 index 00000000000..0cfafe34717 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextFactory.java @@ -0,0 +1,12 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; + +/** + * @author freva + */ +@FunctionalInterface +public interface NodeAgentContextFactory { + NodeAgentContext create(NodeSpec nodeSpec); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 3c34e35ab46..58414ab55f4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -1,14 +1,15 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.ZoneId; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; +import com.yahoo.vespa.hosted.provision.Node; import java.nio.file.FileSystem; import java.nio.file.Path; @@ -25,9 +26,8 @@ public class NodeAgentContextImpl implements NodeAgentContext { private static final Path ROOT = Paths.get("/"); private final String logPrefix; + private final NodeSpec node; private final ContainerName containerName; - private final HostName hostName; - private final NodeType nodeType; private final AthenzService identity; private final DockerNetworking dockerNetworking; private final ZoneId zoneId; @@ -36,13 +36,12 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final String vespaUser; private final String vespaUserOnHost; - public NodeAgentContextImpl(String hostname, NodeType nodeType, AthenzService identity, + public NodeAgentContextImpl(NodeSpec node, AthenzService identity, DockerNetworking dockerNetworking, ZoneId zoneId, Path pathToContainerStorage, Path pathToVespaHome, String vespaUser, String vespaUserOnHost) { - this.hostName = HostName.from(Objects.requireNonNull(hostname)); - this.containerName = ContainerName.fromHostname(hostname); - this.nodeType = Objects.requireNonNull(nodeType); + this.node = Objects.requireNonNull(node); + this.containerName = ContainerName.fromHostname(node.getHostname()); this.identity = Objects.requireNonNull(identity); this.dockerNetworking = Objects.requireNonNull(dockerNetworking); this.zoneId = Objects.requireNonNull(zoneId); @@ -54,18 +53,13 @@ public class NodeAgentContextImpl implements NodeAgentContext { } @Override - public ContainerName containerName() { - return containerName; - } - - @Override - public HostName hostname() { - return hostName; + public NodeSpec node() { + return node; } @Override - public NodeType nodeType() { - return nodeType; + public ContainerName containerName() { + return containerName; } @Override @@ -134,12 +128,25 @@ public class NodeAgentContextImpl implements NodeAgentContext { public void log(Logger logger, Level level, String message, Throwable throwable) { logger.log(level, logPrefix + message, throwable); } - + + @Override + public String toString() { + return "NodeAgentContextImpl{" + + "node=" + node + + ", containerName=" + containerName + + ", identity=" + identity + + ", dockerNetworking=" + dockerNetworking + + ", zoneId=" + zoneId + + ", pathToNodeRootOnHost=" + pathToNodeRootOnHost + + ", pathToVespaHome=" + pathToVespaHome + + ", vespaUser='" + vespaUser + '\'' + + ", vespaUserOnHost='" + vespaUserOnHost + '\'' + + '}'; + } /** For testing only! */ public static class Builder { - private final String hostname; - private NodeType nodeType; + private NodeSpec.Builder nodeSpecBuilder = new NodeSpec.Builder(); private AthenzService identity; private DockerNetworking dockerNetworking; private ZoneId zoneId; @@ -148,12 +155,25 @@ public class NodeAgentContextImpl implements NodeAgentContext { private String vespaUser; private String vespaUserOnHost; + public Builder(NodeSpec node) { + this.nodeSpecBuilder = new NodeSpec.Builder(node); + } + + /** + * Creates a NodeAgentContext.Builder with a NodeSpec that has the given hostname and some + * reasonable values for the remaining required NodeSpec fields. Use {@link #Builder(NodeSpec)} + * if you want to control the entire NodeSpec. + */ public Builder(String hostname) { - this.hostname = hostname; + this.nodeSpecBuilder + .hostname(hostname) + .state(Node.State.active) + .nodeType(NodeType.tenant) + .flavor("d-2-8-50"); } public Builder nodeType(NodeType nodeType) { - this.nodeType = nodeType; + this.nodeSpecBuilder.nodeType(nodeType); return this; } @@ -198,8 +218,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { public NodeAgentContextImpl build() { return new NodeAgentContextImpl( - hostname, - Optional.ofNullable(nodeType).orElse(NodeType.tenant), + nodeSpecBuilder.build(), Optional.ofNullable(identity).orElseGet(() -> new AthenzService("domain", "service")), Optional.ofNullable(dockerNetworking).orElse(DockerNetworking.HOST_NETWORK), Optional.ofNullable(zoneId).orElseGet(() -> new ZoneId(SystemName.dev, Environment.dev, RegionName.defaultName())), diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java new file mode 100644 index 00000000000..54f357d5f29 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java @@ -0,0 +1,102 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import java.time.Clock; +import java.time.Duration; +import java.util.Objects; + +/** + * This class should be used by exactly 2 thread, 1 for each interface it implements. + * + * @author freva + */ +public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAgentScheduler { + + private final Object monitor = new Object(); + private final Clock clock; + + private NodeAgentContext currentContext; + private NodeAgentContext nextContext; + private boolean wantFrozen = false; + private boolean isFrozen = true; + private boolean pendingInterrupt = false; + + public NodeAgentContextManager(Clock clock, NodeAgentContext context) { + this.clock = clock; + this.currentContext = context; + } + + @Override + public void scheduleTickWith(NodeAgentContext context) { + synchronized (monitor) { + nextContext = Objects.requireNonNull(context); + monitor.notifyAll(); // Notify of new context + } + } + + @Override + public boolean setFrozen(boolean frozen, Duration timeout) { + synchronized (monitor) { + if (wantFrozen != frozen) { + wantFrozen = frozen; + monitor.notifyAll(); // Notify the supplier of the wantFrozen change + } + + boolean successful; + long remainder; + long end = clock.instant().plus(timeout).toEpochMilli(); + while (!(successful = isFrozen == frozen) && (remainder = end - clock.millis()) > 0) { + try { + monitor.wait(remainder); // Wait with timeout until the supplier is has reached wanted frozen state + } catch (InterruptedException ignored) { } + } + + return successful; + } + } + + @Override + public NodeAgentContext nextContext() throws InterruptedException { + synchronized (monitor) { + while (setAndGetIsFrozen(wantFrozen) || nextContext == null) { + if (pendingInterrupt) { + pendingInterrupt = false; + throw new InterruptedException("interrupt() was called before next context was scheduled"); + } + + try { + monitor.wait(); // Wait until scheduler provides a new context + } catch (InterruptedException ignored) { } + } + + currentContext = nextContext; + nextContext = null; + return currentContext; + } + } + + @Override + public NodeAgentContext currentContext() { + synchronized (monitor) { + return currentContext; + } + } + + @Override + public void interrupt() { + synchronized (monitor) { + pendingInterrupt = true; + monitor.notifyAll(); + } + } + + private boolean setAndGetIsFrozen(boolean isFrozen) { + synchronized (monitor) { + if (this.isFrozen != isFrozen) { + this.isFrozen = isFrozen; + monitor.notifyAll(); // Notify the scheduler of the isFrozen change + } + return this.isFrozen; + } + } +}
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java new file mode 100644 index 00000000000..1fc730a3cb0 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextSupplier.java @@ -0,0 +1,21 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +/** + * @author freva + */ +public interface NodeAgentContextSupplier { + + /** + * Blocks until the next context is ready + * @return context + * @throws InterruptedException if {@link #interrupt()} was called before this method returned + */ + NodeAgentContext nextContext() throws InterruptedException; + + /** @return the last context returned by {@link #nextContext()} or a default value */ + NodeAgentContext currentContext(); + + /** Interrupts the thread(s) currently waiting in {@link #nextContext()} */ + void interrupt(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java new file mode 100644 index 00000000000..bd13b7eb094 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentFactory.java @@ -0,0 +1,10 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +/** + * @author freva + */ +@FunctionalInterface +public interface NodeAgentFactory { + NodeAgent create(NodeAgentContextSupplier contextSupplier); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 98975dddb56..0bfff82a055 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -17,20 +17,17 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; import com.yahoo.vespa.hosted.provision.Node; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -59,30 +56,21 @@ public class NodeAgentImpl implements NodeAgent { private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName()); - private final Object monitor = new Object(); private final AtomicBoolean terminated = new AtomicBoolean(false); - - private boolean isFrozen = true; - private boolean wantFrozen = false; - private boolean workToDoNow = true; - private boolean expectNodeNotInNodeRepo = false; private boolean hasResumedNode = false; private boolean hasStartedServices = true; - private final NodeAgentContext context; + private final NodeAgentContextSupplier contextSupplier; private final NodeRepository nodeRepository; private final Orchestrator orchestrator; private final DockerOperations dockerOperations; private final StorageMaintainer storageMaintainer; - private final Clock clock; - private final Duration timeBetweenEachConverge; private final Optional<AthenzCredentialsMaintainer> athenzCredentialsMaintainer; private final Optional<AclMaintainer> aclMaintainer; private final Optional<HealthChecker> healthChecker; private int numberOfUnhandledException = 0; private DockerImage imageBeingDownloaded = null; - private Instant lastConverge; private long currentRebootGeneration = 0; private Optional<Long> currentRestartGeneration = Optional.empty(); @@ -115,24 +103,19 @@ public class NodeAgentImpl implements NodeAgent { // Created in NodeAdminImpl public NodeAgentImpl( - final NodeAgentContext context, + final NodeAgentContextSupplier contextSupplier, final NodeRepository nodeRepository, final Orchestrator orchestrator, final DockerOperations dockerOperations, final StorageMaintainer storageMaintainer, - final Clock clock, - final Duration timeBetweenEachConverge, final Optional<AthenzCredentialsMaintainer> athenzCredentialsMaintainer, final Optional<AclMaintainer> aclMaintainer, final Optional<HealthChecker> healthChecker) { - this.context = context; + this.contextSupplier = contextSupplier; this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; this.dockerOperations = dockerOperations; this.storageMaintainer = storageMaintainer; - this.clock = clock; - this.timeBetweenEachConverge = timeBetweenEachConverge; - this.lastConverge = clock.instant(); this.athenzCredentialsMaintainer = athenzCredentialsMaintainer; this.aclMaintainer = aclMaintainer; this.healthChecker = healthChecker; @@ -140,16 +123,15 @@ public class NodeAgentImpl implements NodeAgent { this.loopThread = new Thread(() -> { while (!terminated.get()) { try { - tick(); - } catch (Throwable t) { - numberOfUnhandledException++; - context.log(logger, LogLevel.ERROR, "Unhandled throwable, ignoring", t); - } + NodeAgentContext context = contextSupplier.nextContext(); + converge(context); + } catch (InterruptedException ignored) { } } }); - this.loopThread.setName("tick-" + context.hostname()); + this.loopThread.setName("tick-" + contextSupplier.currentContext().hostname()); this.serviceRestarter = service -> { + NodeAgentContext context = contextSupplier.currentContext(); try { ProcessResult processResult = dockerOperations.executeCommandInContainerAsRoot( context, "service", service, "restart"); @@ -164,46 +146,29 @@ public class NodeAgentImpl implements NodeAgent { } @Override - public boolean setFrozen(boolean frozen) { - synchronized (monitor) { - if (wantFrozen != frozen) { - wantFrozen = frozen; - context.log(logger, LogLevel.DEBUG, wantFrozen ? "Freezing" : "Unfreezing"); - signalWorkToBeDone(); - } - - return isFrozen == frozen; - } - } - - @Override public void start() { - context.log(logger, "Starting with interval " + timeBetweenEachConverge.toMillis() + " ms"); loopThread.start(); } @Override public void stop() { - filebeatRestarter.shutdown(); if (!terminated.compareAndSet(false, true)) { throw new RuntimeException("Can not re-stop a node agent."); } - signalWorkToBeDone(); + filebeatRestarter.shutdown(); + contextSupplier.interrupt(); do { try { loopThread.join(); filebeatRestarter.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - } catch (InterruptedException e) { - context.log(logger, LogLevel.ERROR, - "Interrupted while waiting for converge thread and filebeatRestarter scheduler to shutdown"); - } + } catch (InterruptedException ignored) { } } while (loopThread.isAlive() || !filebeatRestarter.isTerminated()); - context.log(logger, "Stopped"); + contextSupplier.currentContext().log(logger, "Stopped"); } - void startServicesIfNeeded() { + void startServicesIfNeeded(NodeAgentContext context) { if (!hasStartedServices) { context.log(logger, "Starting services"); dockerOperations.startServices(context); @@ -211,10 +176,10 @@ public class NodeAgentImpl implements NodeAgent { } } - void resumeNodeIfNeeded(NodeSpec node) { + void resumeNodeIfNeeded(NodeAgentContext context) { if (!hasResumedNode) { if (!currentFilebeatRestarter.isPresent()) { - storageMaintainer.writeMetricsConfig(context, node); + storageMaintainer.writeMetricsConfig(context); currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay( () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS)); } @@ -225,31 +190,31 @@ public class NodeAgentImpl implements NodeAgent { } } - private void updateNodeRepoWithCurrentAttributes(final NodeSpec node) { + private void updateNodeRepoWithCurrentAttributes(NodeAgentContext context) { final NodeAttributes currentNodeAttributes = new NodeAttributes(); final NodeAttributes newNodeAttributes = new NodeAttributes(); - if (node.getWantedRestartGeneration().isPresent() && - !Objects.equals(node.getCurrentRestartGeneration(), currentRestartGeneration)) { - currentNodeAttributes.withRestartGeneration(node.getCurrentRestartGeneration()); + if (context.node().getWantedRestartGeneration().isPresent() && + !Objects.equals(context.node().getCurrentRestartGeneration(), currentRestartGeneration)) { + currentNodeAttributes.withRestartGeneration(context.node().getCurrentRestartGeneration()); newNodeAttributes.withRestartGeneration(currentRestartGeneration); } - if (!Objects.equals(node.getCurrentRebootGeneration(), currentRebootGeneration)) { - currentNodeAttributes.withRebootGeneration(node.getCurrentRebootGeneration()); + if (!Objects.equals(context.node().getCurrentRebootGeneration(), currentRebootGeneration)) { + currentNodeAttributes.withRebootGeneration(context.node().getCurrentRebootGeneration()); newNodeAttributes.withRebootGeneration(currentRebootGeneration); } - Optional<DockerImage> actualDockerImage = node.getWantedDockerImage().filter(n -> containerState == UNKNOWN); - if (!Objects.equals(node.getCurrentDockerImage(), actualDockerImage)) { - currentNodeAttributes.withDockerImage(node.getCurrentDockerImage().orElse(new DockerImage(""))); + Optional<DockerImage> actualDockerImage = context.node().getWantedDockerImage().filter(n -> containerState == UNKNOWN); + if (!Objects.equals(context.node().getCurrentDockerImage(), actualDockerImage)) { + currentNodeAttributes.withDockerImage(context.node().getCurrentDockerImage().orElse(new DockerImage(""))); newNodeAttributes.withDockerImage(actualDockerImage.orElse(new DockerImage(""))); } - publishStateToNodeRepoIfChanged(currentNodeAttributes, newNodeAttributes); + publishStateToNodeRepoIfChanged(context, currentNodeAttributes, newNodeAttributes); } - private void publishStateToNodeRepoIfChanged(NodeAttributes currentAttributes, NodeAttributes newAttributes) { + private void publishStateToNodeRepoIfChanged(NodeAgentContext context, NodeAttributes currentAttributes, NodeAttributes newAttributes) { if (!currentAttributes.equals(newAttributes)) { context.log(logger, "Publishing new set of attributes to node repo: %s -> %s", currentAttributes, newAttributes); @@ -257,9 +222,9 @@ public class NodeAgentImpl implements NodeAgent { } } - private void startContainer(NodeSpec node) { - ContainerData containerData = createContainerData(context, node); - dockerOperations.createContainer(context, node, containerData); + private void startContainer(NodeAgentContext context) { + ContainerData containerData = createContainerData(context); + dockerOperations.createContainer(context, containerData); dockerOperations.startContainer(context); lastCpuMetric = new CpuUsageReporter(); @@ -268,14 +233,15 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, "Container successfully started, new containerState is " + containerState); } - private Optional<Container> removeContainerIfNeededUpdateContainerState(NodeSpec node, Optional<Container> existingContainer) { + private Optional<Container> removeContainerIfNeededUpdateContainerState( + NodeAgentContext context, Optional<Container> existingContainer) { return existingContainer - .flatMap(container -> removeContainerIfNeeded(node, container)) + .flatMap(container -> removeContainerIfNeeded(context, container)) .map(container -> { - shouldRestartServices(node).ifPresent(restartReason -> { + shouldRestartServices(context.node()).ifPresent(restartReason -> { context.log(logger, "Will restart services: " + restartReason); - restartServices(node, container); - currentRestartGeneration = node.getWantedRestartGeneration(); + restartServices(context, container); + currentRestartGeneration = context.node().getWantedRestartGeneration(); }); return container; }); @@ -292,17 +258,18 @@ public class NodeAgentImpl implements NodeAgent { return Optional.empty(); } - private void restartServices(NodeSpec node, Container existingContainer) { - if (existingContainer.state.isRunning() && node.getState() == Node.State.active) { + private void restartServices(NodeAgentContext context, Container existingContainer) { + if (existingContainer.state.isRunning() && context.node().getState() == Node.State.active) { context.log(logger, "Restarting services"); // Since we are restarting the services we need to suspend the node. - orchestratorSuspendNode(); + orchestratorSuspendNode(context); dockerOperations.restartVespa(context); } } @Override public void stopServices() { + NodeAgentContext context = contextSupplier.currentContext(); context.log(logger, "Stopping services"); if (containerState == ABSENT) return; try { @@ -315,6 +282,7 @@ public class NodeAgentImpl implements NodeAgent { @Override public void suspend() { + NodeAgentContext context = contextSupplier.currentContext(); context.log(logger, "Suspending services on node"); if (containerState == ABSENT) return; try { @@ -358,18 +326,18 @@ public class NodeAgentImpl implements NodeAgent { return Optional.empty(); } - private Optional<Container> removeContainerIfNeeded(NodeSpec node, Container existingContainer) { - Optional<String> removeReason = shouldRemoveContainer(node, existingContainer); + private Optional<Container> removeContainerIfNeeded(NodeAgentContext context, Container existingContainer) { + Optional<String> removeReason = shouldRemoveContainer(context.node(), existingContainer); if (removeReason.isPresent()) { context.log(logger, "Will remove container: " + removeReason.get()); if (existingContainer.state.isRunning()) { - if (node.getState() == Node.State.active) { - orchestratorSuspendNode(); + if (context.node().getState() == Node.State.active) { + orchestratorSuspendNode(context); } try { - if (node.getState() != Node.State.dirty) { + if (context.node().getState() != Node.State.dirty) { suspend(); } stopServices(); @@ -378,9 +346,9 @@ public class NodeAgentImpl implements NodeAgent { } } stopFilebeatSchedulerIfNeeded(); - storageMaintainer.handleCoreDumpsForContainer(context, node, Optional.of(existingContainer)); + storageMaintainer.handleCoreDumpsForContainer(context, Optional.of(existingContainer)); dockerOperations.removeContainer(context, existingContainer); - currentRebootGeneration = node.getWantedRebootGeneration(); + currentRebootGeneration = context.node().getWantedRebootGeneration(); containerState = ABSENT; context.log(logger, "Container successfully removed, new containerState is " + containerState); return Optional.empty(); @@ -399,78 +367,29 @@ public class NodeAgentImpl implements NodeAgent { } } - private void signalWorkToBeDone() { - synchronized (monitor) { - if (!workToDoNow) { - workToDoNow = true; - context.log(logger, LogLevel.DEBUG, "Signaling work to be done"); - monitor.notifyAll(); - } - } - } - - void tick() { - boolean isFrozenCopy; - synchronized (monitor) { - while (!workToDoNow) { - long remainder = timeBetweenEachConverge - .minus(Duration.between(lastConverge, clock.instant())) - .toMillis(); - if (remainder > 0) { - try { - monitor.wait(remainder); - } catch (InterruptedException e) { - context.log(logger, LogLevel.ERROR, "Interrupted while sleeping before tick, ignoring"); - } - } else break; - } - lastConverge = clock.instant(); - workToDoNow = false; - - if (isFrozen != wantFrozen) { - isFrozen = wantFrozen; - context.log(logger, "Updated NodeAgent's frozen state, new value: isFrozen: " + isFrozen); - } - isFrozenCopy = isFrozen; - } - - if (isFrozenCopy) { - context.log(logger, LogLevel.DEBUG, "tick: isFrozen"); - } else { - try { - converge(); - } catch (OrchestratorException | ConvergenceException e) { - context.log(logger, e.getMessage()); - } catch (ContainerNotFoundException e) { - containerState = ABSENT; - context.log(logger, LogLevel.WARNING, "Container unexpectedly gone, resetting containerState to " + containerState); - } catch (DockerException e) { - numberOfUnhandledException++; - context.log(logger, LogLevel.ERROR, "Caught a DockerException", e); - } catch (Exception e) { - numberOfUnhandledException++; - context.log(logger, LogLevel.ERROR, "Unhandled exception, ignoring.", e); - } + public void converge(NodeAgentContext context) { + try { + doConverge(context); + } catch (OrchestratorException | ConvergenceException e) { + context.log(logger, e.getMessage()); + } catch (ContainerNotFoundException e) { + containerState = ABSENT; + context.log(logger, LogLevel.WARNING, "Container unexpectedly gone, resetting containerState to " + containerState); + } catch (DockerException e) { + numberOfUnhandledException++; + context.log(logger, LogLevel.ERROR, "Caught a DockerException", e); + } catch (Throwable e) { + numberOfUnhandledException++; + context.log(logger, LogLevel.ERROR, "Unhandled exception, ignoring", e); } } // Public for testing - void converge() { - final Optional<NodeSpec> optionalNode = nodeRepository.getOptionalNode(context.hostname().value()); - - // We just removed the node from node repo, so this is expected until NodeAdmin stop this NodeAgent - if (!optionalNode.isPresent() && expectNodeNotInNodeRepo) { - context.log(logger, LogLevel.INFO, "Node removed from node repo (as expected)"); - return; - } - - final NodeSpec node = optionalNode.orElseThrow(() -> - new IllegalStateException(String.format("Node '%s' missing from node repository", context.hostname()))); - expectNodeNotInNodeRepo = false; - - Optional<Container> container = getContainer(); + void doConverge(NodeAgentContext context) { + NodeSpec node = context.node(); + Optional<Container> container = getContainer(context); if (!node.equals(lastNode)) { - logChangesToNodeSpec(lastNode, node); + logChangesToNodeSpec(context, lastNode, node); // Current reboot generation uninitialized or incremented from outside to cancel reboot if (currentRebootGeneration < node.getCurrentRebootGeneration()) @@ -485,7 +404,7 @@ public class NodeAgentImpl implements NodeAgent { // Every time the node spec changes, we should clear the metrics for this container as the dimensions // will change and we will be reporting duplicate metrics. if (container.map(c -> c.state.isRunning()).orElse(false)) { - storageMaintainer.writeMetricsConfig(context, node); + storageMaintainer.writeMetricsConfig(context); } lastNode = node; @@ -496,11 +415,11 @@ public class NodeAgentImpl implements NodeAgent { case reserved: case parked: case failed: - removeContainerIfNeededUpdateContainerState(node, container); - updateNodeRepoWithCurrentAttributes(node); + removeContainerIfNeededUpdateContainerState(context, container); + updateNodeRepoWithCurrentAttributes(context); break; case active: - storageMaintainer.handleCoreDumpsForContainer(context, node, container); + storageMaintainer.handleCoreDumpsForContainer(context, container); storageMaintainer.getDiskUsageFor(context) .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / node.getMinDiskAvailableGb()) @@ -512,17 +431,17 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, LogLevel.DEBUG, "Waiting for image to download " + imageBeingDownloaded.asString()); return; } - container = removeContainerIfNeededUpdateContainerState(node, container); + container = removeContainerIfNeededUpdateContainerState(context, container); athenzCredentialsMaintainer.ifPresent(maintainer -> maintainer.converge(context)); if (! container.isPresent()) { containerState = STARTING; - startContainer(node); + startContainer(context); containerState = UNKNOWN; aclMaintainer.ifPresent(AclMaintainer::converge); } - startServicesIfNeeded(); - resumeNodeIfNeeded(node); + startServicesIfNeeded(context); + resumeNodeIfNeeded(context); healthChecker.ifPresent(checker -> checker.verifyHealth(context)); // Because it's more important to stop a bad release from rolling out in prod, @@ -535,32 +454,31 @@ public class NodeAgentImpl implements NodeAgent { // has been successfully rolled out. // - Slobrok and internal orchestrator state is used to determine whether // to allow upgrade (suspend). - updateNodeRepoWithCurrentAttributes(node); + updateNodeRepoWithCurrentAttributes(context); context.log(logger, "Call resume against Orchestrator"); orchestrator.resume(context.hostname().value()); break; case inactive: - removeContainerIfNeededUpdateContainerState(node, container); - updateNodeRepoWithCurrentAttributes(node); + removeContainerIfNeededUpdateContainerState(context, container); + updateNodeRepoWithCurrentAttributes(context); break; case provisioned: nodeRepository.setNodeState(context.hostname().value(), Node.State.dirty); break; case dirty: - removeContainerIfNeededUpdateContainerState(node, container); + removeContainerIfNeededUpdateContainerState(context, container); context.log(logger, "State is " + node.getState() + ", will delete application storage and mark node as ready"); athenzCredentialsMaintainer.ifPresent(maintainer -> maintainer.clearCredentials(context)); storageMaintainer.archiveNodeStorage(context); - updateNodeRepoWithCurrentAttributes(node); + updateNodeRepoWithCurrentAttributes(context); nodeRepository.setNodeState(context.hostname().value(), Node.State.ready); - expectNodeNotInNodeRepo = true; break; default: throw new RuntimeException("UNKNOWN STATE " + node.getState().name()); } } - private void logChangesToNodeSpec(NodeSpec lastNode, NodeSpec node) { + private static void logChangesToNodeSpec(NodeAgentContext context, NodeSpec lastNode, NodeSpec node) { StringBuilder builder = new StringBuilder(); appendIfDifferent(builder, "state", lastNode, node, NodeSpec::getState); if (builder.length() > 0) { @@ -572,7 +490,7 @@ public class NodeAgentImpl implements NodeAgent { return value == null ? "[absent]" : value.toString(); } - private <T> void appendIfDifferent(StringBuilder builder, String name, NodeSpec oldNode, NodeSpec newNode, Function<NodeSpec, T> getter) { + private static <T> void appendIfDifferent(StringBuilder builder, String name, NodeSpec oldNode, NodeSpec newNode, Function<NodeSpec, T> getter) { T oldValue = oldNode == null ? null : getter.apply(oldNode); T newValue = getter.apply(newNode); if (!Objects.equals(oldValue, newValue)) { @@ -592,8 +510,9 @@ public class NodeAgentImpl implements NodeAgent { @SuppressWarnings("unchecked") public void updateContainerNodeMetrics() { - final NodeSpec node = lastNode; - if (node == null || containerState != UNKNOWN) return; + if (containerState != UNKNOWN) return; + final NodeAgentContext context = contextSupplier.currentContext(); + final NodeSpec node = context.node(); Optional<ContainerStats> containerStats = dockerOperations.getContainerStats(context); if (!containerStats.isPresent()) return; @@ -660,10 +579,10 @@ public class NodeAgentImpl implements NodeAgent { metrics.add(networkMetrics); }); - pushMetricsToContainer(metrics); + pushMetricsToContainer(context, metrics); } - private void pushMetricsToContainer(List<DimensionMetrics> metrics) { + private void pushMetricsToContainer(NodeAgentContext context, List<DimensionMetrics> metrics) { StringBuilder params = new StringBuilder(); try { for (DimensionMetrics dimensionMetrics : metrics) { @@ -679,7 +598,7 @@ public class NodeAgentImpl implements NodeAgent { } } - private Optional<Container> getContainer() { + private Optional<Container> getContainer(NodeAgentContext context) { if (containerState == ABSENT) return Optional.empty(); Optional<Container> container = dockerOperations.getContainer(context); if (! container.isPresent()) containerState = ABSENT; @@ -743,12 +662,12 @@ public class NodeAgentImpl implements NodeAgent { // More generally, the node repo response should contain sufficient info on what the docker image is, // to allow the node admin to make decisions that depend on the docker image. Or, each docker image // needs to contain routines for drain and suspend. For many images, these can just be dummy routines. - private void orchestratorSuspendNode() { + private void orchestratorSuspendNode(NodeAgentContext context) { context.log(logger, "Ask Orchestrator for permission to suspend node"); orchestrator.suspend(context.hostname().value()); } - protected ContainerData createContainerData(NodeAgentContext context, NodeSpec node) { + protected ContainerData createContainerData(NodeAgentContext context) { return (pathInContainer, data) -> { throw new UnsupportedOperationException("addFile not implemented"); }; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java new file mode 100644 index 00000000000..540601ffa4f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentScheduler.java @@ -0,0 +1,21 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import java.time.Duration; + +/** + * @author freva + */ +public interface NodeAgentScheduler { + + /** Schedule a tick for NodeAgent to run with the given NodeAgentContext */ + void scheduleTickWith(NodeAgentContext context); + + /** + * Will eventually freeze/unfreeze the node agent + * @param frozen whether node agent should be frozen + * @param timeout maximum duration this method should block while waiting for NodeAgent to reach target state + * @return True if node agent has converged to the desired state + */ + boolean setFrozen(boolean frozen, Duration timeout); +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java new file mode 100644 index 00000000000..c9e4e33f8bb --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java @@ -0,0 +1,41 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.flags; + +import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagData; +import com.yahoo.vespa.flags.json.wire.WireFlagDataList; +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import org.hamcrest.collection.IsMapContaining; +import org.hamcrest.collection.IsMapWithSize; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class RealFlagRepositoryTest { + private final ConfigServerApi configServerApi = mock(ConfigServerApi.class); + private final RealFlagRepository repository = new RealFlagRepository(configServerApi); + + @Test + public void test() { + WireFlagDataList list = new WireFlagDataList(); + list.flags = new ArrayList<>(); + list.flags.add(new WireFlagData()); + list.flags.get(0).id = "id1"; + + when(configServerApi.get(any(), eq(WireFlagDataList.class))).thenReturn(list); + Map<FlagId, FlagData> allFlagData = repository.getAllFlagData(); + assertThat(allFlagData, IsMapWithSize.aMapWithSize(1)); + assertThat(allFlagData, IsMapContaining.hasKey(new FlagId("id1"))); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index cb3a1fb5e2c..109bce4c13f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -15,8 +15,9 @@ import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; import com.yahoo.vespa.hosted.provision.Node; @@ -30,7 +31,6 @@ import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; import java.util.Optional; -import java.util.function.Function; import java.util.logging.Logger; import static com.yahoo.yolean.Exceptions.uncheck; @@ -87,19 +87,20 @@ public class DockerTester implements AutoCloseable { .build(); nodeRepository.updateNodeRepositoryNode(hostSpec); - Clock clock = Clock.systemUTC(); FileSystem fileSystem = TestFileSystem.create(); DockerOperations dockerOperations = new DockerOperationsImpl(docker, processExecuter, ipAddresses); MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl( - new NodeAgentContextImpl.Builder(hostName).fileSystem(fileSystem).build(), nodeRepository, - orchestrator, dockerOperations, storageMaintainer, clock, INTERVAL, Optional.empty(), Optional.empty(), Optional.empty()); - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, Optional.empty(), mr, Clock.systemUTC()); + NodeAgentFactory nodeAgentFactory = contextSupplier -> new NodeAgentImpl( + contextSupplier, nodeRepository, + orchestrator, dockerOperations, storageMaintainer, Optional.empty(), Optional.empty(), Optional.empty()); + NodeAgentContextFactory nodeAgentContextFactory = nodeSpec -> + new NodeAgentContextImpl.Builder(nodeSpec).fileSystem(fileSystem).build(); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, nodeAgentContextFactory, Optional.empty(), mr, Clock.systemUTC()); nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, orchestrator, nodeAdmin, HOST_HOSTNAME); - this.loopThread = new Thread(() -> { + loopThread = new Thread(() -> { nodeAdminStateUpdater.start(); while (! terminated) { @@ -135,8 +136,10 @@ public class DockerTester implements AutoCloseable { @Override public void close() { - terminated = true; + // First, stop NodeAdmin and all the NodeAgents + nodeAdmin.stop(); + terminated = true; do { try { loopThread.join(); @@ -144,8 +147,5 @@ public class DockerTester implements AutoCloseable { e.printStackTrace(); } } while (loopThread.isAlive()); - - // Finally, stop NodeAdmin and all the NodeAgents - nodeAdmin.stop(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 9ea5c87511b..05b9c413594 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -152,12 +152,8 @@ public class StorageMaintainerTest { } private Path executeAs(NodeType nodeType) { - NodeAgentContext context = new NodeAgentContextImpl.Builder("host123-5.test.domain.tld") - .nodeType(nodeType) - .fileSystem(TestFileSystem.create()) - .zoneId(new ZoneId(SystemName.dev, Environment.prod, RegionName.from("us-north-1"))).build(); NodeSpec nodeSpec = new NodeSpec.Builder() - .hostname(context.hostname().value()) + .hostname("host123-5.test.domain.tld") .nodeType(nodeType) .state(Node.State.active) .parentHostname("host123.test.domain.tld") @@ -167,9 +163,12 @@ public class StorageMaintainerTest { .flavor("d-2-8-50") .canonicalFlavor("d-2-8-50") .build(); + NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec) + .fileSystem(TestFileSystem.create()) + .zoneId(new ZoneId(SystemName.dev, Environment.prod, RegionName.from("us-north-1"))).build(); Path path = context.pathOnHostFromPathInNode("/etc/yamas-agent"); uncheck(() -> Files.createDirectories(path)); - storageMaintainer.writeMetricsConfig(context, nodeSpec); + storageMaintainer.writeMetricsConfig(context); return path; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index 3860e2e9780..47e220a968b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -1,22 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; +import com.yahoo.config.provision.NodeType; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; import org.mockito.InOrder; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -31,75 +32,72 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithScheduler; +import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithSchedulerFactory; + /** * @author bakksjo */ public class NodeAdminImplTest { - // Trick to allow mocking of typed interface without casts/warnings. - private interface NodeAgentFactory extends Function<String, NodeAgent> {} - private final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); + + private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory = mock(NodeAgentWithSchedulerFactory.class); + private final NodeAgentContextFactory nodeAgentContextFactory = mock(NodeAgentContextFactory.class); private final ManualClock clock = new ManualClock(); - private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentFactory, Optional.empty(), - new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock); + private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, nodeAgentContextFactory, + Optional.empty(), new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock); @Test public void nodeAgentsAreProperlyLifeCycleManaged() { - final String hostName1 = "host1.test.yahoo.com"; - final String hostName2 = "host2.test.yahoo.com"; - final NodeAgent nodeAgent1 = mock(NodeAgentImpl.class); - final NodeAgent nodeAgent2 = mock(NodeAgentImpl.class); - when(nodeAgentFactory.apply(eq(hostName1))).thenReturn(nodeAgent1); - when(nodeAgentFactory.apply(eq(hostName2))).thenReturn(nodeAgent2); + final NodeSpec nodeSpec1 = createNodeSpec("host1.test.yahoo.com"); + final NodeSpec nodeSpec2 = createNodeSpec("host2.test.yahoo.com"); + final NodeAgentWithScheduler nodeAgent1 = mockNodeAgentWithSchedulerFactory(nodeSpec1); + final NodeAgentWithScheduler nodeAgent2 = mockNodeAgentWithSchedulerFactory(nodeSpec2); + final InOrder inOrder = inOrder(nodeAgentWithSchedulerFactory, nodeAgent1, nodeAgent2); + nodeAdmin.refreshContainersToRun(Collections.emptyList()); + verifyNoMoreInteractions(nodeAgentWithSchedulerFactory); - final InOrder inOrder = inOrder(nodeAgentFactory, nodeAgent1, nodeAgent2); - nodeAdmin.synchronizeNodesToNodeAgents(Collections.emptySet()); - verifyNoMoreInteractions(nodeAgentFactory); - - nodeAdmin.synchronizeNodesToNodeAgents(Collections.singleton(hostName1)); - inOrder.verify(nodeAgentFactory).apply(hostName1); + nodeAdmin.refreshContainersToRun(Collections.singletonList(nodeSpec1)); inOrder.verify(nodeAgent1).start(); + inOrder.verify(nodeAgent2, never()).start(); inOrder.verify(nodeAgent1, never()).stop(); - nodeAdmin.synchronizeNodesToNodeAgents(Collections.singleton(hostName1)); - inOrder.verify(nodeAgentFactory, never()).apply(any(String.class)); + nodeAdmin.refreshContainersToRun(Collections.singletonList(nodeSpec1)); + inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); inOrder.verify(nodeAgent1, never()).start(); inOrder.verify(nodeAgent1, never()).stop(); - nodeAdmin.synchronizeNodesToNodeAgents(Collections.emptySet()); - inOrder.verify(nodeAgentFactory, never()).apply(any(String.class)); + nodeAdmin.refreshContainersToRun(Collections.emptyList()); + inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); verify(nodeAgent1).stop(); - nodeAdmin.synchronizeNodesToNodeAgents(Collections.singleton(hostName2)); - inOrder.verify(nodeAgentFactory).apply(hostName2); + nodeAdmin.refreshContainersToRun(Collections.singletonList(nodeSpec2)); inOrder.verify(nodeAgent2).start(); inOrder.verify(nodeAgent2, never()).stop(); - verify(nodeAgent1).stop(); + inOrder.verify(nodeAgent1, never()).stop(); - nodeAdmin.synchronizeNodesToNodeAgents(Collections.emptySet()); - inOrder.verify(nodeAgentFactory, never()).apply(any(String.class)); + nodeAdmin.refreshContainersToRun(Collections.emptyList()); + inOrder.verify(nodeAgentWithSchedulerFactory, never()).create(any()); inOrder.verify(nodeAgent2, never()).start(); inOrder.verify(nodeAgent2).stop(); - - verifyNoMoreInteractions(nodeAgent1); - verifyNoMoreInteractions(nodeAgent2); + inOrder.verify(nodeAgent1, never()).start(); + inOrder.verify(nodeAgent1, never()).stop(); } @Test public void testSetFrozen() { - List<NodeAgent> nodeAgents = new ArrayList<>(); - Set<String> existingContainerHostnames = new HashSet<>(); + List<NodeSpec> nodeSpecs = new ArrayList<>(); + List<NodeAgentWithScheduler> nodeAgents = new ArrayList<>(); for (int i = 0; i < 3; i++) { - final String hostName = "host" + i + ".test.yahoo.com"; - NodeAgent nodeAgent = mock(NodeAgent.class); - nodeAgents.add(nodeAgent); - when(nodeAgentFactory.apply(eq(hostName))).thenReturn(nodeAgent); + NodeSpec nodeSpec = createNodeSpec("host" + i + ".test.yahoo.com"); + NodeAgentWithScheduler nodeAgent = mockNodeAgentWithSchedulerFactory(nodeSpec); - existingContainerHostnames.add(hostName); + nodeSpecs.add(nodeSpec); + nodeAgents.add(nodeAgent); } - nodeAdmin.synchronizeNodesToNodeAgents(existingContainerHostnames); + nodeAdmin.refreshContainersToRun(nodeSpecs); assertTrue(nodeAdmin.isFrozen()); // Initially everything is frozen to force convergence mockNodeAgentSetFrozenResponse(nodeAgents, true, true, true); @@ -155,10 +153,28 @@ public class NodeAdminImplTest { assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); } - private void mockNodeAgentSetFrozenResponse(List<NodeAgent> nodeAgents, boolean... responses) { + private void mockNodeAgentSetFrozenResponse(List<NodeAgentWithScheduler> nodeAgents, boolean... responses) { for (int i = 0; i < nodeAgents.size(); i++) { - NodeAgent nodeAgent = nodeAgents.get(i); - when(nodeAgent.setFrozen(anyBoolean())).thenReturn(responses[i]); + NodeAgentWithScheduler nodeAgent = nodeAgents.get(i); + when(nodeAgent.setFrozen(anyBoolean(), any())).thenReturn(responses[i]); } } + + private NodeSpec createNodeSpec(String hostname) { + return new NodeSpec.Builder() + .hostname(hostname) + .state(Node.State.active) + .nodeType(NodeType.tenant) + .flavor("default") + .build(); + } + + private NodeAgentWithScheduler mockNodeAgentWithSchedulerFactory(NodeSpec nodeSpec) { + NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec).build(); + when(nodeAgentContextFactory.create(eq(nodeSpec))).thenReturn(context); + + NodeAgentWithScheduler nodeAgentWithScheduler = mock(NodeAgentWithScheduler.class); + when(nodeAgentWithSchedulerFactory.create(eq(context))).thenReturn(nodeAgentWithScheduler); + return nodeAgentWithScheduler; + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java index 437195ca6d5..74ba5561c8e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -42,7 +42,7 @@ public class NodeAdminStateUpdaterTest { private final NodeAdmin nodeAdmin = mock(NodeAdmin.class); private final HostName hostHostname = HostName.from("basehost1.test.yahoo.com"); - private final NodeAdminStateUpdater refresher = spy(new NodeAdminStateUpdater( + private final NodeAdminStateUpdater updater = spy(new NodeAdminStateUpdater( nodeRepository, orchestrator, nodeAdmin, hostHostname)); @@ -58,19 +58,19 @@ public class NodeAdminStateUpdaterTest { { // Initially everything is frozen to force convergence - assertResumeStateError(RESUMED, "NodeAdmin is not yet unfrozen"); + assertConvergeError(RESUMED, "NodeAdmin is not yet unfrozen"); when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); - refresher.converge(RESUMED); + updater.converge(RESUMED); verify(orchestrator, times(1)).resume(hostHostname.value()); // We are already resumed, so this should return without resuming again - refresher.converge(RESUMED); + updater.converge(RESUMED); verify(orchestrator, times(1)).resume(hostHostname.value()); verify(nodeAdmin, times(2)).setFrozen(eq(false)); // Lets try to suspend node admin only when(nodeAdmin.setFrozen(eq(true))).thenReturn(false); - assertResumeStateError(SUSPENDED_NODE_ADMIN, "NodeAdmin is not yet frozen"); + assertConvergeError(SUSPENDED_NODE_ADMIN, "NodeAdmin is not yet frozen"); verify(nodeAdmin, times(2)).setFrozen(eq(false)); } @@ -81,10 +81,10 @@ public class NodeAdminStateUpdaterTest { when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); doThrow(new RuntimeException(exceptionMessage)).doNothing() .when(orchestrator).suspend(eq(hostHostname.value())); - assertResumeStateError(SUSPENDED_NODE_ADMIN, exceptionMessage); + assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMessage); verify(nodeAdmin, times(2)).setFrozen(eq(false)); - refresher.converge(SUSPENDED_NODE_ADMIN); + updater.converge(SUSPENDED_NODE_ADMIN); verify(nodeAdmin, times(2)).setFrozen(eq(false)); } @@ -93,13 +93,13 @@ public class NodeAdminStateUpdaterTest { final String exceptionMessage = "Failed to stop services"; verify(orchestrator, times(0)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); doThrow(new RuntimeException(exceptionMessage)).doNothing().when(nodeAdmin).stopNodeAgentServices(eq(activeHostnames)); - assertResumeStateError(SUSPENDED, exceptionMessage); + assertConvergeError(SUSPENDED, exceptionMessage); verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(suspendHostnames)); // Make sure we dont roll back if we fail to stop services - we will try to stop again next tick verify(nodeAdmin, times(2)).setFrozen(eq(false)); // Finally we are successful in transitioning to frozen - refresher.converge(SUSPENDED); + updater.converge(SUSPENDED); } } @@ -110,31 +110,38 @@ public class NodeAdminStateUpdaterTest { // Initially everything is frozen to force convergence when(nodeAdmin.setFrozen(eq(false))).thenReturn(true); - refresher.converge(RESUMED); + updater.converge(RESUMED); verify(nodeAdmin, times(1)).setFrozen(eq(false)); + verify(nodeAdmin, times(1)).refreshContainersToRun(any()); // Let's start suspending, we are able to freeze the nodes, but orchestrator denies suspension when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofSeconds(1)); when(nodeAdmin.setFrozen(eq(true))).thenReturn(true); doThrow(new RuntimeException(exceptionMsg)).when(orchestrator).suspend(eq(hostHostname.value())); - assertResumeStateError(SUSPENDED_NODE_ADMIN, exceptionMsg); + assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); verify(nodeAdmin, times(1)).setFrozen(eq(true)); - assertResumeStateError(SUSPENDED_NODE_ADMIN, exceptionMsg); + verify(orchestrator, times(1)).suspend(eq(hostHostname.value())); + assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); verify(nodeAdmin, times(2)).setFrozen(eq(true)); - assertResumeStateError(SUSPENDED_NODE_ADMIN, exceptionMsg); + verify(orchestrator, times(2)).suspend(eq(hostHostname.value())); + assertConvergeError(SUSPENDED_NODE_ADMIN, exceptionMsg); verify(nodeAdmin, times(3)).setFrozen(eq(true)); - verify(nodeAdmin, times(1)).setFrozen(eq(false)); // No new unfreezes during last 2 ticks + verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); + + // No new unfreezes nor refresh while trying to freeze + verify(nodeAdmin, times(1)).setFrozen(eq(false)); verify(nodeAdmin, times(1)).refreshContainersToRun(any()); // Only resume and fetch containers when subsystem freeze duration expires when(nodeAdmin.subsystemFreezeDuration()).thenReturn(Duration.ofHours(1)); - assertResumeStateError(SUSPENDED_NODE_ADMIN, exceptionMsg); + assertConvergeError(SUSPENDED_NODE_ADMIN, "Timed out trying to freeze all nodes: will force an unfrozen tick"); verify(nodeAdmin, times(2)).setFrozen(eq(false)); + verify(orchestrator, times(3)).suspend(eq(hostHostname.value())); // no new suspend calls verify(nodeAdmin, times(2)).refreshContainersToRun(any()); // We change our mind, want to remain resumed - refresher.converge(RESUMED); + updater.converge(RESUMED); verify(nodeAdmin, times(3)).setFrozen(eq(false)); // Make sure that we unfreeze! } @@ -146,24 +153,24 @@ public class NodeAdminStateUpdaterTest { // Resume and suspend only require that node-agents are frozen and permission from // orchestrator to resume/suspend host. Therefore, if host is not active, we only need to freeze. - refresher.converge(RESUMED); + updater.converge(RESUMED); verify(orchestrator, never()).resume(eq(hostHostname.value())); - refresher.converge(SUSPENDED_NODE_ADMIN); + updater.converge(SUSPENDED_NODE_ADMIN); verify(orchestrator, never()).suspend(eq(hostHostname.value())); // When doing batch suspend, only suspend the containers if the host is not active List<String> activeHostnames = nodeRepository.getNodes(hostHostname.value()).stream() .map(NodeSpec::getHostname) .collect(Collectors.toList()); - refresher.converge(SUSPENDED); + updater.converge(SUSPENDED); verify(orchestrator, times(1)).suspend(eq(hostHostname.value()), eq(activeHostnames)); } - private void assertResumeStateError(NodeAdminStateUpdater.State targetState, String reason) { + private void assertConvergeError(NodeAdminStateUpdater.State targetState, String reason) { try { - refresher.converge(targetState); - fail("Expected set resume state to fail with \"" + reason + "\", but it succeeded without error"); + updater.converge(targetState); + fail("Expected converging to " + targetState + " to fail with \"" + reason + "\", but it succeeded without error"); } catch (RuntimeException e) { assertEquals(reason, e.getMessage()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java new file mode 100644 index 00000000000..f32e3d91e34 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -0,0 +1,142 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import org.junit.Test; + +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author freva + */ +public class NodeAgentContextManagerTest { + + private static final int TIMEOUT = 10_000; + + private final Clock clock = Clock.systemUTC(); + private final NodeAgentContext initialContext = generateContext(); + private final NodeAgentContextManager manager = new NodeAgentContextManager(clock, initialContext); + + @Test(timeout = TIMEOUT) + public void returns_immediately_if_next_context_is_ready() throws InterruptedException { + NodeAgentContext context1 = generateContext(); + manager.scheduleTickWith(context1); + + assertSame(initialContext, manager.currentContext()); + assertSame(context1, manager.nextContext()); + assertSame(context1, manager.currentContext()); + } + + @Test(timeout = TIMEOUT) + public void blocks_in_nextContext_until_one_is_scheduled() throws InterruptedException { + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + assertFalse(async.response.isPresent()); + Thread.sleep(10); + assertFalse(async.response.isPresent()); + + NodeAgentContext context1 = generateContext(); + manager.scheduleTickWith(context1); + + async.awaitResult(); + assertEquals(Optional.of(context1), async.response); + assertFalse(async.exception.isPresent()); + } + + @Test(timeout = TIMEOUT) + public void blocks_in_nextContext_until_interrupt() throws InterruptedException { + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + assertFalse(async.response.isPresent()); + Thread.sleep(10); + assertFalse(async.response.isPresent()); + + manager.interrupt(); + + async.awaitResult(); + assertEquals(Optional.of(InterruptedException.class), async.exception.map(Exception::getClass)); + assertFalse(async.response.isPresent()); + } + + @Test(timeout = TIMEOUT) + public void setFrozen_does_not_block_with_no_timeout() throws InterruptedException { + assertFalse(manager.setFrozen(false, Duration.ZERO)); + + // Generate new context and get it from the supplier, this completes the unfreeze + NodeAgentContext context1 = generateContext(); + manager.scheduleTickWith(context1); + assertSame(context1, manager.nextContext()); + + assertTrue(manager.setFrozen(false, Duration.ZERO)); + } + + @Test(timeout = TIMEOUT) + public void setFrozen_blocks_at_least_for_duration_of_timeout() { + long wantedDurationMillis = 100; + long start = clock.millis(); + assertFalse(manager.setFrozen(false, Duration.ofMillis(wantedDurationMillis))); + long actualDurationMillis = clock.millis() - start; + + assertTrue(actualDurationMillis >= wantedDurationMillis); + } + + @Test(timeout = TIMEOUT) + public void setFrozen_is_successful_if_converged_in_time() throws InterruptedException { + AsyncExecutor<Boolean> async = new AsyncExecutor<>(() -> manager.setFrozen(false, Duration.ofMillis(500))); + + assertFalse(async.response.isPresent()); + + NodeAgentContext context1 = generateContext(); + manager.scheduleTickWith(context1); + assertSame(context1, manager.nextContext()); + + async.awaitResult(); + assertEquals(Optional.of(true), async.response); + assertFalse(async.exception.isPresent()); + } + + private static NodeAgentContext generateContext() { + return new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); + } + + private class AsyncExecutor<T> { + private final Object monitor = new Object(); + private final Thread thread; + private volatile Optional<T> response = Optional.empty(); + private volatile Optional<Exception> exception = Optional.empty(); + private boolean completed = false; + + private AsyncExecutor(ThrowingSupplier<T> supplier) { + this.thread = new Thread(() -> { + try { + response = Optional.of(supplier.get()); + } catch (Exception e) { + exception = Optional.of(e); + } + synchronized (monitor) { + completed = true; + monitor.notifyAll(); + } + }); + this.thread.start(); + } + + private void awaitResult() { + synchronized (monitor) { + while (!completed) { + try { + monitor.wait(); + } catch (InterruptedException ignored) { } + } + } + } + } + + private interface ThrowingSupplier<T> { + T get() throws Exception; + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index b6128fc8693..e392ac34414 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.provision.NodeType; import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.dockerapi.Container; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; @@ -28,7 +29,6 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.Duration; import java.util.Collections; import java.util.Map; import java.util.Optional; @@ -36,8 +36,6 @@ import java.util.Set; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -56,14 +54,21 @@ import static org.mockito.Mockito.when; * @author Øyvind Bakksjø */ public class NodeAgentImplTest { - private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); private static final double MIN_CPU_CORES = 2; private static final double MIN_MAIN_MEMORY_AVAILABLE_GB = 16; private static final double MIN_DISK_AVAILABLE_GB = 250; private static final String vespaVersion = "1.2.3"; private final String hostName = "host1.test.yahoo.com"; - private final NodeAgentContext context = new NodeAgentContextImpl.Builder(hostName).build(); + private final NodeSpec.Builder nodeBuilder = new NodeSpec.Builder() + .hostname(hostName) + .nodeType(NodeType.tenant) + .flavor("docker") + .minCpuCores(MIN_CPU_CORES) + .minMainMemoryAvailableGb(MIN_MAIN_MEMORY_AVAILABLE_GB) + .minDiskAvailableGb(MIN_DISK_AVAILABLE_GB); + + private final NodeAgentContextSupplier contextSupplier = mock(NodeAgentContextSupplier.class); private final DockerImage dockerImage = new DockerImage("dockerImage"); private final DockerOperations dockerOperations = mock(DockerOperations.class); private final NodeRepository nodeRepository = mock(NodeRepository.class); @@ -76,16 +81,6 @@ public class NodeAgentImplTest { Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); private final AthenzCredentialsMaintainer athenzCredentialsMaintainer = mock(AthenzCredentialsMaintainer.class); - private final ManualClock clock = new ManualClock(); - - private final NodeSpec.Builder nodeBuilder = new NodeSpec.Builder() - .hostname(context.hostname().value()) - .nodeType(NodeType.tenant) - .flavor("docker") - .minCpuCores(MIN_CPU_CORES) - .minMainMemoryAvailableGb(MIN_MAIN_MEMORY_AVAILABLE_GB) - .minDiskAvailableGb(MIN_DISK_AVAILABLE_GB); - @Test public void upToDateContainerIsUntouched() { @@ -97,11 +92,12 @@ public class NodeAgentImplTest { .vespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L)); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).suspend(any(String.class)); @@ -125,11 +121,12 @@ public class NodeAgentImplTest { .vespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(217432719360L)); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(context)); } @@ -145,27 +142,28 @@ public class NodeAgentImplTest { .vespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L)); - nodeAgent.converge(); + nodeAgent.doConverge(context); inOrder.verify(dockerOperations, never()).startServices(eq(context)); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); nodeAgent.suspend(); - nodeAgent.converge(); + nodeAgent.doConverge(context); inOrder.verify(dockerOperations, never()).startServices(eq(context)); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); // Expect a resume, but no start services // No new suspends/stops, so no need to resume/start - nodeAgent.converge(); + nodeAgent.doConverge(context); inOrder.verify(dockerOperations, never()).startServices(eq(context)); inOrder.verify(dockerOperations, never()).resumeNode(eq(context)); nodeAgent.suspend(); nodeAgent.stopServices(); - nodeAgent.converge(); + nodeAgent.doConverge(context); inOrder.verify(dockerOperations, times(1)).startServices(eq(context)); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); } @@ -181,13 +179,14 @@ public class NodeAgentImplTest { .currentRestartGeneration(restartGeneration.get()) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(dockerOperations, never()).startServices(any()); @@ -195,7 +194,7 @@ public class NodeAgentImplTest { final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer, healthChecker); inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage)); - inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), any()); inOrder.verify(dockerOperations, times(1)).startContainer(eq(context)); inOrder.verify(aclMaintainer, times(1)).converge(); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); @@ -216,13 +215,14 @@ public class NodeAgentImplTest { .vespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(orchestrator, never()).suspend(any(String.class)); verify(orchestrator, never()).resume(any(String.class)); @@ -241,29 +241,25 @@ public class NodeAgentImplTest { .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion); + NodeAgentContext firstContext = createContext(specBuilder.build()); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); - NodeSpec firstSpec = specBuilder.build(); - NodeSpec secondSpec = specBuilder.minDiskAvailableGb(200).build(); - NodeSpec thirdSpec = specBuilder.minCpuCores(4).build(); - - when(nodeRepository.getOptionalNode(hostName)) - .thenReturn(Optional.of(firstSpec)) - .thenReturn(Optional.of(secondSpec)) - .thenReturn(Optional.of(thirdSpec)); + when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); + when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); - nodeAgent.converge(); - nodeAgent.converge(); - nodeAgent.converge(); + nodeAgent.doConverge(firstContext); + NodeAgentContext secondContext = createContext(specBuilder.minDiskAvailableGb(200).build()); + nodeAgent.doConverge(secondContext); + NodeAgentContext thirdContext = createContext(specBuilder.minCpuCores(4).build()); + nodeAgent.doConverge(thirdContext); InOrder inOrder = inOrder(orchestrator, dockerOperations); inOrder.verify(orchestrator).resume(any(String.class)); inOrder.verify(orchestrator).resume(any(String.class)); inOrder.verify(orchestrator).suspend(any(String.class)); - inOrder.verify(dockerOperations).removeContainer(eq(context), any()); - inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(thirdSpec), any()); - inOrder.verify(dockerOperations).startContainer(eq(context)); + inOrder.verify(dockerOperations).removeContainer(eq(thirdContext), any()); + inOrder.verify(dockerOperations, times(1)).createContainer(eq(thirdContext), any()); + inOrder.verify(dockerOperations).startContainer(eq(thirdContext)); inOrder.verify(orchestrator).resume(any(String.class)); } @@ -281,14 +277,16 @@ public class NodeAgentImplTest { .currentRestartGeneration(currentRestartGeneration) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + doThrow(new OrchestratorException("Denied")).when(orchestrator).suspend(eq(hostName)); try { - nodeAgent.converge(); + nodeAgent.doConverge(context); fail("Expected to throw an exception"); - } catch (Exception ignored) { } + } catch (OrchestratorException ignored) { } - verify(dockerOperations, never()).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, never()).createContainer(eq(context), any()); verify(dockerOperations, never()).startContainer(eq(context)); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(any(String.class), any(NodeAttributes.class)); @@ -308,6 +306,7 @@ public class NodeAgentImplTest { .currentRebootGeneration(currentRebootGeneration) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); @@ -317,22 +316,22 @@ public class NodeAgentImplTest { .when(healthChecker).verifyHealth(eq(context)); try { - nodeAgent.converge(); + nodeAgent.doConverge(context); } catch (ConvergenceException ignored) {} // First time we fail to resume because health verification fails verify(orchestrator, times(1)).suspend(eq(hostName)); verify(dockerOperations, times(1)).removeContainer(eq(context), any()); - verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(1)).createContainer(eq(context), any()); verify(dockerOperations, times(1)).startContainer(eq(context)); verify(orchestrator, never()).resume(eq(hostName)); verify(nodeRepository, never()).updateNodeAttributes(any(), any()); - nodeAgent.converge(); + nodeAgent.doConverge(context); // Do not reboot the container again verify(dockerOperations, times(1)).removeContainer(eq(context), any()); - verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(1)).createContainer(eq(context), any()); verify(orchestrator, times(1)).resume(eq(hostName)); verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() .withRebootGeneration(wantedRebootGeneration))); @@ -348,11 +347,12 @@ public class NodeAgentImplTest { .vespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).resume(any(String.class)); @@ -365,18 +365,19 @@ public class NodeAgentImplTest { .state(Node.State.ready) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(null,false); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - nodeAgent.converge(); - nodeAgent.converge(); - nodeAgent.converge(); + nodeAgent.doConverge(context); + nodeAgent.doConverge(context); + nodeAgent.doConverge(context); // Should only be called once, when we initialize verify(dockerOperations, times(1)).getContainer(eq(context)); verify(dockerOperations, never()).removeContainer(eq(context), any()); - verify(dockerOperations, never()).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, never()).createContainer(eq(context), any()); verify(dockerOperations, never()).startContainer(eq(context)); verify(orchestrator, never()).resume(any(String.class)); verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); @@ -392,11 +393,12 @@ public class NodeAgentImplTest { .vespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - nodeAgent.converge(); + nodeAgent.doConverge(context); final InOrder inOrder = inOrder(storageMaintainer, dockerOperations); inOrder.verify(dockerOperations, never()).removeContainer(eq(context), any()); @@ -413,11 +415,12 @@ public class NodeAgentImplTest { .wantedVespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(nodeRepository, never()).updateNodeAttributes(eq(hostName), any()); } @@ -433,20 +436,21 @@ public class NodeAgentImplTest { .state(nodeState) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - nodeAgent.converge(); + nodeAgent.doConverge(context); final InOrder inOrder = inOrder(storageMaintainer, dockerOperations, nodeRepository); inOrder.verify(dockerOperations, times(1)).stopServices(eq(context)); - inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), eq(node), any()); + inOrder.verify(storageMaintainer, times(1)).handleCoreDumpsForContainer(eq(context), any()); inOrder.verify(dockerOperations, times(1)).removeContainer(eq(context), any()); inOrder.verify(storageMaintainer, times(1)).archiveNodeStorage(eq(context)); inOrder.verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(Node.State.ready)); - verify(dockerOperations, never()).createContainer(eq(context), any(), any()); + verify(dockerOperations, never()).createContainer(eq(context), any()); verify(dockerOperations, never()).startContainer(eq(context)); verify(dockerOperations, never()).suspendNode(eq(context)); verify(dockerOperations, times(1)).stopServices(eq(context)); @@ -474,10 +478,11 @@ public class NodeAgentImplTest { .state(Node.State.provisioned) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(Node.State.dirty)); } @@ -490,15 +495,16 @@ public class NodeAgentImplTest { .vespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); - nodeAgent.tick(); + nodeAgent.doConverge(context); verify(dockerOperations, times(1)).removeContainer(eq(context), any()); - verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(1)).createContainer(eq(context), any()); verify(dockerOperations, times(1)).startContainer(eq(context)); } @@ -511,6 +517,7 @@ public class NodeAgentImplTest { .vespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); @@ -523,7 +530,7 @@ public class NodeAgentImplTest { // 1st try try { - nodeAgent.converge(); + nodeAgent.doConverge(context); fail("Expected to throw an exception"); } catch (RuntimeException ignored) { } @@ -531,7 +538,7 @@ public class NodeAgentImplTest { inOrder.verifyNoMoreInteractions(); // 2nd try - nodeAgent.converge(); + nodeAgent.doConverge(context); inOrder.verify(dockerOperations).resumeNode(any()); inOrder.verify(orchestrator).resume(hostName); @@ -539,33 +546,6 @@ public class NodeAgentImplTest { } @Test - public void testSetFrozen() { - NodeAgentImpl nodeAgent = spy(makeNodeAgent(dockerImage, true)); - doNothing().when(nodeAgent).converge(); - - nodeAgent.tick(); - verify(nodeAgent, times(1)).converge(); - - assertFalse(nodeAgent.setFrozen(true)); // Returns true because we are not frozen until tick is called - nodeAgent.tick(); - verify(nodeAgent, times(1)).converge(); // Frozen should be set, therefore converge is never called - - assertTrue(nodeAgent.setFrozen(true)); // Attempt to set frozen again, but it's already set - clock.advance(Duration.ofSeconds(35)); // workToDoNow is no longer set, so we need to wait the regular time - nodeAgent.tick(); - verify(nodeAgent, times(1)).converge(); - - assertFalse(nodeAgent.setFrozen(false)); // Unfreeze, but still need to call tick for it to take effect - nodeAgent.tick(); - verify(nodeAgent, times(2)).converge(); - - assertTrue(nodeAgent.setFrozen(false)); - clock.advance(Duration.ofSeconds(35)); // workToDoNow is no longer set, so we need to wait the regular time - nodeAgent.tick(); - verify(nodeAgent, times(3)).converge(); - } - - @Test public void start_container_subtask_failure_leads_to_container_restart() { final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) @@ -573,30 +553,30 @@ public class NodeAgentImplTest { .wantedVespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false)); - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context)); try { - nodeAgent.converge(); + nodeAgent.doConverge(context); fail("Expected to get DockerException"); } catch (DockerException ignored) { } verify(dockerOperations, never()).removeContainer(eq(context), any()); - verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(1)).createContainer(eq(context), any()); verify(dockerOperations, times(1)).startContainer(eq(context)); verify(nodeAgent, never()).resumeNodeIfNeeded(any()); // The docker container was actually started and is running, but subsequent exec calls to set up // networking failed mockGetContainer(dockerImage, true); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(dockerOperations, times(1)).removeContainer(eq(context), any()); - verify(dockerOperations, times(2)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(2)).createContainer(eq(context), any()); verify(dockerOperations, times(2)).startContainer(eq(context)); verify(nodeAgent, times(1)).resumeNodeIfNeeded(any()); } @@ -631,6 +611,7 @@ public class NodeAgentImplTest { .parentHostname("parent.host.name.yahoo.com") .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); @@ -638,12 +619,9 @@ public class NodeAgentImplTest { when(dockerOperations.getContainerStats(eq(context))) .thenReturn(Optional.of(stats1)) .thenReturn(Optional.of(stats2)); - - nodeAgent.converge(); // Run the converge loop once to initialize lastNode + nodeAgent.updateContainerNodeMetrics(); // Update metrics once to init and lastCpuMetric - clock.advance(Duration.ofSeconds(1234)); - Path pathToExpectedMetrics = Paths.get(classLoader.getResource("expected.container.system.metrics.txt").getPath()); String expectedMetrics = new String(Files.readAllBytes(pathToExpectedMetrics)) .replaceAll("\\s", "") @@ -674,13 +652,11 @@ public class NodeAgentImplTest { .state(Node.State.ready) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); when(dockerOperations.getContainerStats(eq(context))).thenReturn(Optional.empty()); - nodeAgent.converge(); // Run the converge loop once to initialize lastNode - nodeAgent.updateContainerNodeMetrics(); Set<Map<String, Object>> actualMetrics = metricReceiver.getAllMetricsRaw(); @@ -696,20 +672,21 @@ public class NodeAgentImplTest { .wantedVespaVersion(vespaVersion) .build(); + NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(null, false); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); - nodeAgent.converge(); + nodeAgent.doConverge(context); verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).suspend(any(String.class)); final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer); inOrder.verify(dockerOperations, times(1)).pullImageAsyncIfNeeded(eq(dockerImage)); - inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + inOrder.verify(dockerOperations, times(1)).createContainer(eq(context), any()); inOrder.verify(dockerOperations, times(1)).startContainer(eq(context)); inOrder.verify(aclMaintainer, times(1)).converge(); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); @@ -722,24 +699,33 @@ public class NodeAgentImplTest { mockGetContainer(dockerImage, isRunning); when(dockerOperations.getContainerStats(any())).thenReturn(Optional.of(emptyContainerStats)); - doNothing().when(storageMaintainer).writeMetricsConfig(any(), any()); + doNothing().when(storageMaintainer).writeMetricsConfig(any()); - return new NodeAgentImpl(context, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.of(athenzCredentialsMaintainer), Optional.of(aclMaintainer), + return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations, + storageMaintainer, Optional.of(athenzCredentialsMaintainer), Optional.of(aclMaintainer), Optional.of(healthChecker)); } private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { - Optional<Container> container = dockerImage != null ? - Optional.of(new Container( - hostName, - dockerImage, - ContainerResources.from(MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB), - context.containerName(), - isRunning ? Container.State.RUNNING : Container.State.EXITED, - isRunning ? 1 : 0)) : - Optional.empty(); - - when(dockerOperations.getContainer(eq(context))).thenReturn(container); + doAnswer(invoc -> { + NodeAgentContext context = invoc.getArgument(0); + if (!hostName.equals(context.hostname().value())) + throw new IllegalArgumentException(); + return dockerImage != null ? + Optional.of(new Container( + hostName, + dockerImage, + ContainerResources.from(MIN_CPU_CORES, MIN_MAIN_MEMORY_AVAILABLE_GB), + ContainerName.fromHostname(hostName), + isRunning ? Container.State.RUNNING : Container.State.EXITED, + isRunning ? 1 : 0)) : + Optional.empty(); + }).when(dockerOperations).getContainer(any()); + } + + private NodeAgentContext createContext(NodeSpec nodeSpec) { + NodeAgentContext context = new NodeAgentContextImpl.Builder(nodeSpec).build(); + when(contextSupplier.currentContext()).thenReturn(context); + return context; } } diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml index 9276ce0e7c9..22ab615bfad 100644 --- a/node-repository/src/main/config/node-repository.xml +++ b/node-repository/src/main/config/node-repository.xml @@ -11,5 +11,10 @@ <binding>https://*/nodes/v2/*</binding> </handler> +<handler id="com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler" bundle="node-repository"> + <binding>http://*/loadbalancers/v1/*</binding> + <binding>https://*/loadbalancers/v1/*</binding> +</handler> + <preprocess:include file="node-flavors.xml" required="false" /> <preprocess:include file="node-repository-config.xml" required="false" /> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 7e518ee1728..442013b8a6a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; -import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; @@ -77,7 +77,7 @@ public final class Node { Objects.requireNonNull(history, "A null node history is not permitted"); Objects.requireNonNull(type, "A null node type is not permitted"); - this.ipAddresses = ImmutableSortedSet.copyOf(IP.naturalOrder, ipAddresses); + this.ipAddresses = ImmutableSet.copyOf(ipAddresses); this.ipAddressPool = new IP.AddressPool(this, ipAddressPool); this.hostname = hostname; this.parentHostname = parentHostname; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index effc5b1a41d..4ac3a839ae1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.provision.maintenance.LoadBalancerExpirer; import java.util.List; import java.util.Objects; @@ -50,8 +51,8 @@ public class LoadBalancer { } /** - * Returns whether this load balancer is inactive. Inactive load balancers cannot be reactivated, and are - * eventually deleted + * Returns whether this load balancer is inactive. Inactive load balancers cannot be re-activated, and are + * eventually removed by {@link LoadBalancerExpirer}. */ public boolean inactive() { return inactive; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index b589e5aed2f..a5a0d8cb2f8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -17,6 +17,10 @@ public class LoadBalancerServiceMock implements LoadBalancerService { private final Map<LoadBalancerId, LoadBalancer> loadBalancers = new HashMap<>(); + public Map<LoadBalancerId, LoadBalancer> loadBalancers() { + return Collections.unmodifiableMap(loadBalancers); + } + @Override public Protocol protocol() { return Protocol.ipv4; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 9d87a835960..68d597fb839 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -108,9 +108,12 @@ public class InfrastructureProvisioner extends Maintainer { } private void removeApplication(ApplicationId applicationId) { - NestedTransaction nestedTransaction = new NestedTransaction(); - provisioner.remove(nestedTransaction, applicationId); - nestedTransaction.commit(); - duperModel.infraApplicationRemoved(applicationId); + // Use the DuperModel as source-of-truth on whether it has also been activated (to avoid periodic removals) + if (duperModel.infraApplicationIsActive(applicationId)) { + NestedTransaction nestedTransaction = new NestedTransaction(); + provisioner.remove(nestedTransaction, applicationId); + nestedTransaction.commit(); + duperModel.infraApplicationRemoved(applicationId); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java new file mode 100644 index 00000000000..4b66dff3032 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -0,0 +1,83 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Periodically remove inactive load balancers permanently. + * + * When an application is removed, any associated load balancers are only deactivated. This maintainer ensures that + * such resources are eventually freed. + * + * @author mpolden + */ +public class LoadBalancerExpirer extends Maintainer { + + private final LoadBalancerService service; + private final CuratorDatabaseClient db; + + public LoadBalancerExpirer(NodeRepository nodeRepository, Duration interval, JobControl jobControl, + LoadBalancerService service) { + super(nodeRepository, interval, jobControl); + this.service = Objects.requireNonNull(service, "service must be non-null"); + this.db = nodeRepository.database(); + } + + @Override + protected void maintain() { + removeInactive(); + } + + private void removeInactive() { + List<LoadBalancerId> failed = new ArrayList<>(); + Exception lastException = null; + try (Lock lock = db.lockLoadBalancers()) { + for (LoadBalancerId loadBalancer : inactiveLoadBlancers()) { + if (hasNodes(loadBalancer.application())) { // Defer removal if there are still nodes allocated to application + continue; + } + try { + service.remove(loadBalancer); + db.removeLoadBalancer(loadBalancer); + } catch (Exception e) { + failed.add(loadBalancer); + lastException = e; + } + } + } + if (!failed.isEmpty()) { + log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", + failed.size(), + failed.stream() + .map(LoadBalancerId::serializedForm) + .collect(Collectors.joining(", ")), + interval()), + lastException); + } + } + + private boolean hasNodes(ApplicationId application) { + return !nodeRepository().getNodes(application).isEmpty(); + } + + private List<LoadBalancerId> inactiveLoadBlancers() { + return db.readLoadBalancers().entrySet().stream() + .filter(entry -> entry.getValue().inactive()) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java index f5576ae00fc..49ede9962eb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java @@ -56,8 +56,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { try { if (jobControl.isActive(name())) maintain(); - } - catch (RuntimeException e) { + } catch (Throwable e) { log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 946c43ca8fc..2bc60de3c8d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetireIPv4OnlyNodes; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicyList; @@ -50,6 +51,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeRetirer nodeRetirer; private final MetricsReporter metricsReporter; private final InfrastructureProvisioner infrastructureProvisioner; + private final LoadBalancerExpirer loadBalancerExpirer; private final JobControl jobControl; private final InfrastructureVersions infrastructureVersions; @@ -59,15 +61,17 @@ public class NodeRepositoryMaintenance extends AbstractComponent { HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator, Metric metric, ConfigserverConfig configserverConfig, - DuperModelInfraApi duperModelInfraApi) { + DuperModelInfraApi duperModelInfraApi, + LoadBalancerService loadBalancerService) { this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), - orchestrator, metric, configserverConfig, duperModelInfraApi); + orchestrator, metric, configserverConfig, duperModelInfraApi, loadBalancerService); } public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner, HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, - ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi) { + ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi, + LoadBalancerService loadBalancerService) { DefaultTimes defaults = new DefaultTimes(zone); jobControl = new JobControl(nodeRepository.database()); infrastructureVersions = new InfrastructureVersions(nodeRepository.database()); @@ -84,6 +88,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl); metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl); infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl, duperModelInfraApi); + loadBalancerExpirer = new LoadBalancerExpirer(nodeRepository, durationFromEnv("load_balancer_expiry").orElse(defaults.loadBalancerExpiry), jobControl, loadBalancerService); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintain(); @@ -109,6 +114,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { provisionedExpirer.deconstruct(); metricsReporter.deconstruct(); infrastructureProvisioner.deconstruct(); + loadBalancerExpirer.deconstruct(); } public JobControl jobControl() { return jobControl; } @@ -156,6 +162,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration metricsInterval; private final Duration retiredInterval; private final Duration infrastructureProvisionInterval; + private final Duration loadBalancerExpiry; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -171,6 +178,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { metricsInterval = Duration.ofMinutes(1); infrastructureProvisionInterval = Duration.ofMinutes(3); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; + loadBalancerExpiry = Duration.ofHours(1); if (zone.environment().equals(Environment.prod) && zone.system() != SystemName.cd) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 074a20fc82d..fbc358893e1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; -import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; import com.google.common.primitives.UnsignedBytes; import com.yahoo.vespa.hosted.provision.Node; @@ -59,7 +59,7 @@ public class IP { public AddressPool(Node owner, Set<String> addresses) { this.owner = Objects.requireNonNull(owner, "owner must be non-null"); - this.addresses = ImmutableSortedSet.copyOf(naturalOrder, requireAddresses(addresses)); + this.addresses = ImmutableSet.copyOf(requireAddresses(addresses)); } /** @@ -200,9 +200,9 @@ public class IP { /** All IP addresses in this */ public Set<String> addresses() { - ImmutableSortedSet.Builder<String> builder = ImmutableSortedSet.orderedBy(naturalOrder); - builder.add(ipv6Address); + ImmutableSet.Builder<String> builder = ImmutableSet.builder(); ipv4Address.ifPresent(builder::add); + builder.add(ipv6Address); return builder.build(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java index 2ceffc54dd5..2715b1131b3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java @@ -36,7 +36,7 @@ public class CuratorDatabase { private final CuratorCounter changeGenerationCounter; /** A partial cache of the Curator database, which is only valid if generations match */ - private final AtomicReference<CuratorDatabaseCache> cache = new AtomicReference<>(); + private final AtomicReference<Cache> cache = new AtomicReference<>(); /** Whether we should return data from the cache or always read fro ZooKeeper */ private final boolean useCache; @@ -110,12 +110,12 @@ public class CuratorDatabase { // the data to read is protected by a lock which is held now, and during any writes of the data. /** Returns the immediate, local names of the children under this node in any order */ - List<String> getChildren(Path path) { return getCache().getChildren(path); } + List<String> getChildren(Path path) { return getSession().getChildren(path); } - Optional<byte[]> getData(Path path) { return getCache().getData(path); } + Optional<byte[]> getData(Path path) { return getSession().getData(path); } /** Invalidates the current cache if outdated. */ - private CuratorDatabaseCache getCache() { + Session getSession() { if (changeGenerationCounter.get() != cache.get().generation) synchronized (cacheCreationLock) { while (changeGenerationCounter.get() != cache.get().generation) @@ -126,8 +126,8 @@ public class CuratorDatabase { } /** Caches must only be instantiated using this method */ - private CuratorDatabaseCache newCache(long generation) { - return useCache ? new CuratorDatabaseCache(generation, curator) : new DeactivatedCache(generation, curator); + private Cache newCache(long generation) { + return useCache ? new Cache(generation, curator) : new NoCache(generation, curator); } /** @@ -135,10 +135,10 @@ public class CuratorDatabase { * This is merely a recording of what Curator returned at various points in time when * it had the counter at this generation. */ - private static class CuratorDatabaseCache { + private static class Cache implements Session { private final long generation; - + /** The curator instance used to fetch missing data */ protected final Curator curator; @@ -149,23 +149,17 @@ public class CuratorDatabase { private final Map<Path, Optional<byte[]>> data = new ConcurrentHashMap<>(); /** Create an empty snapshot at a given generation (as an empty snapshot is a valid partial snapshot) */ - private CuratorDatabaseCache(long generation, Curator curator) { + private Cache(long generation, Curator curator) { this.generation = generation; this.curator = curator; } - public long generation() { return generation; } - - /** - * Returns the children of this path, which may be empty. - */ + @Override public List<String> getChildren(Path path) { return children.computeIfAbsent(path, key -> ImmutableList.copyOf(curator.getChildren(path))); } - /** - * Returns the a copy of the content of this child - which may be empty. - */ + @Override public Optional<byte[]> getData(Path path) { return data.computeIfAbsent(path, key -> curator.getData(path)).map(data -> Arrays.copyOf(data, data.length)); } @@ -173,9 +167,9 @@ public class CuratorDatabase { } /** An implementation of the curator database cache which does no caching */ - private static class DeactivatedCache extends CuratorDatabaseCache { - - private DeactivatedCache(long generation, Curator curator) { super(generation, curator); } + private static class NoCache extends Cache { + + private NoCache(long generation, Curator curator) { super(generation, curator); } @Override public List<String> getChildren(Path path) { return curator.getChildren(path); } @@ -185,4 +179,17 @@ public class CuratorDatabase { } + interface Session { + + /** + * Returns the children of this path, which may be empty. + */ + List<String> getChildren(Path path); + + /** + * Returns the a copy of the content of this child - which may be empty. + */ + Optional<byte[]> getData(Path path); + + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index c4031f3ccba..da4d2a0afb2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -251,9 +251,10 @@ public class CuratorDatabaseClient { List<Node> nodes = new ArrayList<>(); if (states.length == 0) states = Node.State.values(); + CuratorDatabase.Session session = curatorDatabase.getSession(); for (Node.State state : states) { - for (String hostname : curatorDatabase.getChildren(toPath(state))) { - Optional<Node> node = getNode(hostname, state); + for (String hostname : session.getChildren(toPath(state))) { + Optional<Node> node = getNode(session, hostname, state); node.ifPresent(nodes::add); // node might disappear between getChildren and getNode } } @@ -270,21 +271,29 @@ public class CuratorDatabaseClient { return nodes; } - /** + /** * Returns a particular node, or empty if this noe is not in any of the given states. * If no states are given this returns the node if it is present in any state. */ - public Optional<Node> getNode(String hostname, Node.State ... states) { + public Optional<Node> getNode(CuratorDatabase.Session session, String hostname, Node.State ... states) { if (states.length == 0) states = Node.State.values(); for (Node.State state : states) { - Optional<byte[]> nodeData = curatorDatabase.getData(toPath(state, hostname)); + Optional<byte[]> nodeData = session.getData(toPath(state, hostname)); if (nodeData.isPresent()) return nodeData.map((data) -> nodeSerializer.fromJson(state, data)); } return Optional.empty(); } + /** + * Returns a particular node, or empty if this noe is not in any of the given states. + * If no states are given this returns the node if it is present in any state. + */ + public Optional<Node> getNode(String hostname, Node.State ... states) { + return getNode(curatorDatabase.getSession(), hostname, states); + } + private Path toPath(Node.State nodeState) { return root.append(toDir(nodeState)); } private Path toPath(Node node) { @@ -449,10 +458,10 @@ public class CuratorDatabaseClient { }); } - public void removeLoadBalancer(LoadBalancer loadBalancer) { + public void removeLoadBalancer(LoadBalancerId loadBalancer) { NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer.id()).getAbsolute())); + curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer).getAbsolute())); transaction.commit(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 1c640a6f074..f96ecc0431a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Status; import java.io.IOException; @@ -141,7 +142,7 @@ public class NodeSerializer { } private void toSlime(Set<String> ipAddresses, Cursor array) { - ipAddresses.forEach(array::addString); + ipAddresses.stream().sorted(IP.naturalOrder).forEach(array::addString); } // ---------------- Deserialization -------------------------------------------------- diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java index 483f19ed5b0..f3d8f42f3b7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.provision.maintenance.JobControl; import java.io.IOException; import java.io.OutputStream; -import java.net.URI; +import java.util.TreeSet; /** A response containing maintenance job status */ public class JobsResponse extends HttpResponse { @@ -25,13 +25,12 @@ public class JobsResponse extends HttpResponse { public void render(OutputStream stream) throws IOException { Slime slime = new Slime(); Cursor root = slime.setObject(); - Cursor jobArray = root.setArray("jobs"); - for (String jobName : jobControl.jobs()) + for (String jobName : new TreeSet<>(jobControl.jobs())) jobArray.addObject().setString("name", jobName); Cursor inactiveArray = root.setArray("inactive"); - for (String jobName : jobControl.inactiveJobs()) + for (String jobName : new TreeSet<>(jobControl.inactiveJobs())) inactiveArray.addString(jobName); new JsonFormat(true).encode(stream, slime); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java new file mode 100644 index 00000000000..6ffac2c0fbc --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.vespa.hosted.provision.NoSuchNodeException; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.yolean.Exceptions; + +import javax.inject.Inject; +import java.util.logging.Level; + +/** + * @author mpolden + */ +public class LoadBalancersApiHandler extends LoggingRequestHandler { + + private final NodeRepository nodeRepository; + + @Inject + public LoadBalancersApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) { + super(parentCtx); + this.nodeRepository = nodeRepository; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } + catch (NotFoundException | NoSuchNodeException e) { + return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); + } + catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + String path = request.getUri().getPath(); + if (path.equals("/loadbalancers/v1/")) return new LoadBalancersResponse(request, nodeRepository); + throw new NotFoundException("Nothing at path '" + path + "'"); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java new file mode 100644 index 00000000000..04a1cdaeeda --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java @@ -0,0 +1,78 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * @author mpolden + */ +public class LoadBalancersResponse extends HttpResponse { + + private final NodeRepository nodeRepository; + private final HttpRequest request; + + public LoadBalancersResponse(HttpRequest request, NodeRepository nodeRepository) { + super(200); + this.request = request; + this.nodeRepository = nodeRepository; + } + + private Optional<ApplicationId> application() { + return Optional.ofNullable(request.getProperty("application")).map(ApplicationFilter::toApplicationId); + } + + private List<LoadBalancer> loadBalancers() { + return application().map(nodeRepository.database()::readLoadBalancers) + .orElseGet(() -> new ArrayList<>(nodeRepository.database().readLoadBalancers().values())); + } + + @Override + public String getContentType() { return "application/json"; } + + @Override + public void render(OutputStream stream) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor loadBalancerArray = root.setArray("loadBalancers"); + + loadBalancers().forEach(lb -> { + Cursor lbObject = loadBalancerArray.addObject(); + lbObject.setString("id", lb.id().serializedForm()); + lbObject.setString("application", lb.id().application().application().value()); + lbObject.setString("tenant", lb.id().application().tenant().value()); + lbObject.setString("instance", lb.id().application().instance().value()); + lbObject.setString("cluster", lb.id().cluster().value()); + lbObject.setString("hostname", lb.hostname().value()); + + Cursor portArray = lbObject.setArray("ports"); + lb.ports().forEach(portArray::addLong); + + Cursor realArray = lbObject.setArray("reals"); + lb.reals().forEach(real -> { + Cursor realObject = realArray.addObject(); + realObject.setString("hostname", real.hostname().value()); + realObject.setString("ipAddress", real.ipAddress()); + realObject.setLong("port", real.port()); + }); + + lbObject.setBool("inactive", lb.inactive()); + }); + + new JsonFormat(true).encode(stream, slime); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 733f5df7858..d2ab3c20080 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -6,10 +6,10 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.History; @@ -81,7 +81,7 @@ class NodesResponse extends HttpResponse { @Override public void render(OutputStream stream) throws IOException { - stream.write(toJson()); + new JsonFormat(true).encode(stream, slime); } @Override @@ -89,10 +89,6 @@ class NodesResponse extends HttpResponse { return "application/json"; } - private byte[] toJson() throws IOException { - return SlimeUtils.toJsonBytes(slime); - } - private void statesToSlime(Cursor root) { Cursor states = root.setObject("states"); for (Node.State state : Node.State.values()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java index 95f69dc1c2a..6225a8a4fc4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java @@ -24,32 +24,52 @@ import java.util.stream.Collectors; /** * Authorizer for config server REST APIs. This contains the rules for all API paths where the authorization process - * requires information from the node-repository to make a decision + * may require information from the node-repository to make a decision * * @author mpolden * @author bjorncs */ public class Authorizer implements BiPredicate<NodePrincipal, URI> { - private final NodeRepository nodeRepository; private final Set<String> whitelistedHostnames; + private final AthenzIdentity controllerIdentity; + private final AthenzIdentity configServerIdentity = new AthenzService("vespa.vespa", "configserver"); + private final AthenzIdentity proxyIdentity = new AthenzService("vespa.vespa", "proxy"); + private final AthenzIdentity tenantIdentity = new AthenzService("vespa.vespa", "tenant-host"); private final Set<AthenzIdentity> trustedIdentities; + private final Set<AthenzIdentity> hostAdminIdentities; // TODO Remove whitelisted hostnames as these nodes should be included through 'trustedIdentities' public Authorizer(SystemName system, NodeRepository nodeRepository, Set<String> whitelistedHostnames) { this.nodeRepository = nodeRepository; this.whitelistedHostnames = whitelistedHostnames; - this.trustedIdentities = getTrustedIdentities(system); + controllerIdentity = system == SystemName.main + ? new AthenzService("vespa.vespa", "hosting") + : new AthenzService("vespa.vespa.cd", "hosting"); + this.trustedIdentities = new HashSet<>(Arrays.asList(controllerIdentity, configServerIdentity)); + this.hostAdminIdentities = new HashSet<>(Arrays.asList(controllerIdentity, configServerIdentity, proxyIdentity, tenantIdentity)); } /** Returns whether principal is authorized to access given URI */ @Override public boolean test(NodePrincipal principal, URI uri) { - // Trusted services can access everything - if (principal.getAthenzIdentityName().isPresent() - && trustedIdentities.contains(principal.getAthenzIdentityName().get())) { - return true; + if (principal.getAthenzIdentityName().isPresent()) { + // All host admins can retrieve flags data + if (uri.getPath().equals("/flags/v1/data") || uri.getPath().equals("/flags/v1/data/")) { + return hostAdminIdentities.contains(principal.getAthenzIdentityName().get()); + } + + // Only controller can access everything else in flags + if (uri.getPath().startsWith("/flags/v1/")) { + return principal.getAthenzIdentityName().get().equals(controllerIdentity); + } + + // Trusted services can access everything + if (trustedIdentities.contains(principal.getAthenzIdentityName().get())) { + return true; + } } + if (principal.getHostname().isPresent()) { String hostname = principal.getHostname().get(); if (isAthenzProviderApi(uri)) { @@ -108,18 +128,6 @@ public class Authorizer implements BiPredicate<NodePrincipal, URI> { return !resources.isEmpty() && resources.stream().anyMatch(resource -> predicate.test(resource, principal)); } - - private static Set<AthenzIdentity> getTrustedIdentities(SystemName system) { - Set<AthenzIdentity> trustedIdentities = new HashSet<>(); - trustedIdentities.add(new AthenzService("vespa.vespa", "configserver")); - AthenzService controllerIdentity = - system == SystemName.main - ? new AthenzService("vespa.vespa", "hosting") - : new AthenzService("vespa.vespa.cd", "hosting"); - trustedIdentities.add(controllerIdentity); - return trustedIdentities; - } - private Optional<Node> getNode(String hostname) { // Ignore potential path traversal. Node repository happily passes arguments unsanitized all the way down to // curator... diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index 110d0ca94d0..bc8772af952 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -9,31 +9,34 @@ package com.yahoo.vespa.hosted.provision.testutils; */ public class ContainerConfig { - public static String servicesXmlV2(int port) { - return "<jdisc version='1.0'>\n" + - " <config name=\"container.handler.threadpool\">\n" + - " <maxthreads>10</maxthreads>\n" + - " </config> \n" + - " <component id='com.yahoo.test.ManualClock'/>\n" + - " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" + - " <component id='com.yahoo.config.provision.Zone'/>\n" + - " <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler'>\n" + - " <binding>http://*/nodes/v2/*</binding>\n" + - " </handler>\n" + - " <http>\n" + - " <server id='myServer' port='" + port + "'/>\n" + - " </http>\n" + - "</jdisc>"; - } + public static String servicesXmlV2(int port) { + return "<jdisc version='1.0'>\n" + + " <config name=\"container.handler.threadpool\">\n" + + " <maxthreads>10</maxthreads>\n" + + " </config> \n" + + " <component id='com.yahoo.test.ManualClock'/>\n" + + " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" + + " <component id='com.yahoo.config.provision.Zone'/>\n" + + " <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler'>\n" + + " <binding>http://*/nodes/v2/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler'>\n" + + " <binding>http://*/loadbalancers/v1/*</binding>\n" + + " </handler>\n" + + " <http>\n" + + " <server id='myServer' port='" + port + "'/>\n" + + " </http>\n" + + "</jdisc>"; + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 1b8ae58a97d..183255db06b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -18,6 +18,7 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Status; @@ -111,6 +112,8 @@ public class MockNodeRepository extends NodeRepository { dirtyRecursively("host55.yahoo.com", Agent.system, getClass().getSimpleName()); ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp")); + // TODO: Remove this once feature flag is removed + this.flags().setEnabled(FlagId.exclusiveLoadBalancer, zoneApp, true); ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java index bc83e3525ad..4fd20d6991b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java @@ -78,6 +78,7 @@ public class InfrastructureProvisionerTest { public void remove_application_if_without_target_version() { when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.empty()); addNode(1, Node.State.active, Optional.of(target)); + when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(true); infrastructureProvisioner.maintain(); verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId()); verifyRemoved(1); @@ -85,12 +86,26 @@ public class InfrastructureProvisionerTest { @Test public void remove_application_if_without_nodes() { + remove_application_without_nodes(true); + } + + @Test + public void skip_remove_unless_active() { + remove_application_without_nodes(false); + } + + private void remove_application_without_nodes(boolean applicationIsActive) { when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target)); addNode(1, Node.State.failed, Optional.of(target)); addNode(2, Node.State.parked, Optional.empty()); + when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(applicationIsActive); infrastructureProvisioner.maintain(); - verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId()); - verifyRemoved(1); + if (applicationIsActive) { + verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId()); + verifyRemoved(1); + } else { + verifyRemoved(0); + } } @Test @@ -199,6 +214,7 @@ public class InfrastructureProvisionerTest { @Test public void avoid_provisioning_if_no_usable_nodes() { when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target)); + when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(true); infrastructureProvisioner.maintain(); verifyRemoved(1); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java new file mode 100644 index 00000000000..59323bfdeb5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -0,0 +1,96 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Vtag; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.Zone; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.flag.FlagId; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class LoadBalancerExpirerTest { + + private ProvisioningTester tester; + + @Before + public void before() { + tester = new ProvisioningTester(Zone.defaultZone()); + } + + @Test + public void test_maintain() { + LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), + Duration.ofDays(1), + new JobControl(tester.nodeRepository().database()), + tester.loadBalancerService()); + tester.nodeRepository().flags().setEnabled(FlagId.exclusiveLoadBalancer, true); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + + // Deploy two applications with load balancers + ClusterSpec.Id cluster = ClusterSpec.Id.from("qrs"); + ApplicationId app1 = tester.makeApplicationId(); + ApplicationId app2 = tester.makeApplicationId(); + LoadBalancerId lb1 = new LoadBalancerId(app1, cluster); + LoadBalancerId lb2 = new LoadBalancerId(app2, cluster); + deployApplication(app1, cluster); + deployApplication(app2, cluster); + assertEquals(2, loadBalancers.get().size()); + + // Remove one application deactivates load balancers for that application + removeApplication(app1); + assertTrue(loadBalancers.get().get(lb1).inactive()); + assertFalse(loadBalancers.get().get(lb2).inactive()); + + // Expirer defers removal while nodes are still allocated to application + expirer.maintain(); + assertEquals(2, tester.loadBalancerService().loadBalancers().size()); + + // Expirer removes load balancers once nodes are deallocated + dirtyNodesOf(app1); + expirer.maintain(); + assertFalse("Inactive load balancer removed", tester.loadBalancerService().loadBalancers().containsKey(lb1)); + + // Active load balancer is left alone + assertFalse(loadBalancers.get().get(lb2).inactive()); + assertTrue("Active load balancer is not removed", tester.loadBalancerService().loadBalancers().containsKey(lb2)); + } + + private void dirtyNodesOf(ApplicationId application) { + tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, "unit-test"); + } + + private void removeApplication(ApplicationId application) { + NestedTransaction transaction = new NestedTransaction(); + tester.provisioner().remove(transaction, application); + transaction.commit(); + } + + private void deployApplication(ApplicationId application, ClusterSpec.Id cluster) { + tester.makeReadyNodes(10, "default"); + List<HostSpec> hosts = tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster, + Vtag.currentVersion, false), + 2, 1, + "default"); + tester.activate(application, hosts); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 5612c8dc665..2c63b9fd62c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -101,7 +101,7 @@ public class ProvisioningTester { } } - static FlavorsConfig createConfig() { + public static FlavorsConfig createConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3); b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2); @@ -170,11 +170,11 @@ public class ProvisioningTester { deactivateTransaction.commit(); } - Collection<String> toHostNames(Collection<HostSpec> hosts) { + public Collection<String> toHostNames(Collection<HostSpec> hosts) { return hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); } - Set<String> toHostNames(List<Node> nodes) { + public Set<String> toHostNames(List<Node> nodes) { return nodes.stream().map(Node::hostname).collect(Collectors.toSet()); } @@ -182,7 +182,7 @@ public class ProvisioningTester { * Asserts that each active node in this application has a restart count equaling the * number of matches to the given filters */ - void assertRestartCount(ApplicationId application, HostFilter... filters) { + public void assertRestartCount(ApplicationId application, HostFilter... filters) { for (Node node : nodeRepository.getNodes(application, Node.State.active)) { int expectedRestarts = 0; for (HostFilter filter : filters) @@ -199,7 +199,7 @@ public class ProvisioningTester { assertEquals(beforeFailCount + 1, failedNode.status().failCount()); } - void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) { + public void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) { Set<Integer> indices = new HashSet<>(); for (HostSpec host : hosts) { ClusterSpec nodeCluster = host.membership().get().cluster(); @@ -214,14 +214,14 @@ public class ProvisioningTester { assertEquals("Indexes in " + requestedCluster + " are disjunct", hosts.size(), indices.size()); } - HostSpec removeOne(Set<HostSpec> hosts) { + public HostSpec removeOne(Set<HostSpec> hosts) { Iterator<HostSpec> i = hosts.iterator(); HostSpec removed = i.next(); i.remove(); return removed; } - ApplicationId makeApplicationId() { + public ApplicationId makeApplicationId() { return ApplicationId.from( TenantName.from(UUID.randomUUID().toString()), ApplicationName.from(UUID.randomUUID().toString()), @@ -232,15 +232,15 @@ public class ProvisioningTester { return makeReadyNodes(n, flavor, NodeType.tenant); } - List<Node> makeReadyNodes(int n, String flavor, NodeType type) { + public List<Node> makeReadyNodes(int n, String flavor, NodeType type) { return makeReadyNodes(n, flavor, type, 0); } - List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) { + public List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) { return makeProvisionedNodes(count, flavor, type, ipAddressPoolSize, false); } - List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { + public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { List<Node> nodes = new ArrayList<>(n); for (int i = 0; i < n; i++) { @@ -290,7 +290,7 @@ public class ProvisioningTester { return nodes; } - List<Node> makeConfigServers(int n, String flavor, Version configServersVersion) { + public List<Node> makeConfigServers(int n, String flavor, Version configServersVersion) { List<Node> nodes = new ArrayList<>(n); MockNameResolver nameResolver = (MockNameResolver)nodeRepository().nameResolver(); @@ -322,33 +322,33 @@ public class ProvisioningTester { return nodeRepository.getNodes(application.getApplicationId(), Node.State.active); } - List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) { + public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) { return makeReadyNodes(n, flavor, type, ipAddressPoolSize, false); } - List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { + public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { List<Node> nodes = makeProvisionedNodes(n, flavor, type, ipAddressPoolSize, dualStack); nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); } /** Creates a set of virtual docker nodes on a single docker host */ - List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) { + public List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) { return makeReadyVirtualNodes(n, flavor, Optional.of(dockerHostId)); } /** Creates a set of virtual nodes on a single parent host */ - List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) { + public List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) { return makeReadyVirtualNodes(n, 0, flavor, parentHostId, index -> UUID.randomUUID().toString()); } /** Creates a set of virtual nodes on a single parent host */ - List<Node> makeReadyVirtualNode(int index, String flavor, String parentHostId) { + public List<Node> makeReadyVirtualNode(int index, String flavor, String parentHostId) { return makeReadyVirtualNodes(1, index, flavor, Optional.of(parentHostId), i -> String.format("node%03d", i)); } /** Creates a set of virtual nodes on a single parent host */ - List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId, + public List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId, Function<Integer, String> nodeNamer) { List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < count + startIndex; i++) { @@ -362,16 +362,16 @@ public class ProvisioningTester { return nodes; } - List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) { + public List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) { return makeReadyVirtualNodes(n, flavor, Optional.of(parentHostId)); } /** Returns the hosts from the input list which are not retired */ - List<HostSpec> nonRetired(Collection<HostSpec> hosts) { + public List<HostSpec> nonRetired(Collection<HostSpec> hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); } - void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) { + public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) { long actualNodesWithFlavor = hostSpecs.stream() .map(HostSpec::hostname) .map(this::getNodeFlavor) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index ae7f3f14975..ec09805ff5d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -706,6 +706,13 @@ public class RestApiTest { } } + @Test + public void test_load_balancers() throws Exception { + assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); + assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=zoneapp.zoneapp.zoneapp"), "load-balancers.json"); + assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); + } + private String asDockerNodeJson(String hostname, String parentHostname, int additionalIpCount, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + "\"," + createIpAddresses(ipAddress) + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java index 38128e66861..5e643bd09ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java @@ -139,6 +139,30 @@ public class AuthorizerTest { } @Test + public void flags_authorization() { + // Tenant nodes cannot access flags resources + assertFalse(authorizedTenantNode("node1", "/flags/v1/data")); + assertFalse(authorizedTenantNode("node1", "/flags/v1/data/flagid")); + assertFalse(authorizedTenantNode("node1", "/flags/v1/foo")); + + // Host node can access data + assertTrue(authorizedTenantHostNode("host1", "/flags/v1/data")); + assertFalse(authorizedTenantHostNode("host1", "/flags/v1/data/flagid")); + assertFalse(authorizedTenantHostNode("host1", "/flags/v1/foo")); + assertTrue(authorizedTenantHostNode("proxy1-host", "/flags/v1/data")); + assertFalse(authorizedTenantHostNode("proxy1-host", "/flags/v1/data/flagid")); + assertFalse(authorizedTenantHostNode("proxy1-host", "/flags/v1/foo")); + assertTrue(authorizedController("vespa.vespa.configserver", "/flags/v1/data")); + assertFalse(authorizedController("vespa.vespa.configserver", "/flags/v1/data/flagid")); + assertFalse(authorizedController("vespa.vespa.configserver", "/flags/v1/foo")); + + // Controller can access everything + assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/data")); + assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/data/flagid")); + assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/foo")); + } + + @Test public void routing_authorization() { // Node of proxy or proxyhost type can access routing resource assertFalse(authorizedTenantNode("node1", "/routing/v1/status")); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json index 8fd09b4a274..a606777e9fd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json @@ -4,7 +4,7 @@ "id": "exclusive-load-balancer", "enabled": false, "enabledHostnames": [], - "enabledApplications": [] + "enabledApplications": ["zoneapp:zoneapp:zoneapp"] } ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json index 78de52e4e85..4baf75f2169 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json @@ -7,6 +7,7 @@ "host1" ], "enabledApplications": [ + "zoneapp:zoneapp:zoneapp", "foo:bar:default" ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json new file mode 100644 index 00000000000..c882f7652d8 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json @@ -0,0 +1,43 @@ +{ + "loadBalancers": [ + { + "id": "zoneapp:zoneapp:zoneapp:node-admin", + "application": "zoneapp", + "tenant": "zoneapp", + "instance": "zoneapp", + "cluster": "node-admin", + "hostname": "lb-zoneapp.zoneapp.zoneapp-node-admin", + "ports": [ + 4443 + ], + "reals": [ + { + "hostname": "dockerhost4.yahoo.com", + "ipAddress": "127.0.0.1", + "port": 4443 + }, + { + "hostname": "dockerhost5.yahoo.com", + "ipAddress": "127.0.0.1", + "port": 4443 + }, + { + "hostname": "dockerhost2.yahoo.com", + "ipAddress": "127.0.0.1", + "port": 4443 + }, + { + "hostname": "dockerhost3.yahoo.com", + "ipAddress": "127.0.0.1", + "port": 4443 + }, + { + "hostname": "dockerhost1.yahoo.com", + "ipAddress": "127.0.0.1", + "port": 4443 + } + ], + "inactive": false + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index 99cb9fd91f5..1432d2f4ea5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -1,46 +1,49 @@ { - "jobs":[ + "jobs": [ { - "name":"PeriodicApplicationMaintainer" + "name": "DirtyExpirer" }, { - "name":"FailedExpirer" + "name": "FailedExpirer" }, { - "name":"ReservationExpirer" + "name": "InactiveExpirer" }, { - "name":"RetiredExpirer" + "name": "InfrastructureProvisioner" }, { - "name":"NodeRebooter" + "name": "LoadBalancerExpirer" }, { - "name":"InactiveExpirer" + "name": "MetricsReporter" }, { - "name":"DirtyExpirer" + "name": "NodeFailer" }, { - "name":"NodeRetirer" + "name": "NodeRebooter" }, { - "name":"OperatorChangeApplicationMaintainer" + "name": "NodeRetirer" }, { - "name":"ProvisionedExpirer" + "name": "OperatorChangeApplicationMaintainer" }, { - "name":"MetricsReporter" + "name": "PeriodicApplicationMaintainer" }, { - "name":"InfrastructureProvisioner" + "name": "ProvisionedExpirer" }, { - "name":"NodeFailer" + "name": "ReservationExpirer" + }, + { + "name": "RetiredExpirer" } ], - "inactive":[ + "inactive": [ "NodeFailer" ] } diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index db6116073e6..c0c706383f6 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -106,7 +106,7 @@ public: TuneFileSummary(), _fileHeaderContext, _noTlSyncer, - NULL), + nullptr), _serialNum(1) { } @@ -262,7 +262,7 @@ public: op.setPrevDbDocumentId(prevDbdId); _ddb->getFeedHandler().storeOperation(op, std::make_shared<search::IgnoreCallback>()); SearchView *sv(dynamic_cast<SearchView *>(_ddb->getReadySubDB()->getSearchView().get())); - if (sv != NULL) { + if (sv != nullptr) { // cf. FeedView::putAttributes() DocIdLimit &docIdLimit = sv->getDocIdLimit(); if (docIdLimit.get() <= lid) @@ -278,14 +278,11 @@ private: ResultConfig _resultCfg; std::set<vespalib::string> _markupFields; - const ResultConfig &getResultConfig() const - { + const ResultConfig &getResultConfig() const{ return _resultCfg; } - const std::set<vespalib::string> & - getMarkupFields() const - { + const std::set<vespalib::string> &getMarkupFields() const{ return _markupFields; } @@ -295,31 +292,12 @@ private: GeneralResultPtr getResult(const DocsumReply & reply, uint32_t id, uint32_t resultClassID); - bool - assertString(const std::string & exp, - const std::string & fieldName, - DocumentStoreAdapter &dsa, - uint32_t id); + bool assertString(const std::string & exp, const std::string & fieldName, DocumentStoreAdapter &dsa, uint32_t id); - bool - assertString(const std::string &exp, - const std::string &fieldName, - const DocsumReply &reply, - uint32_t id, - uint32_t resultClassID); + void assertTensor(const Tensor::UP &exp, const std::string &fieldName, const DocsumReply &reply, + uint32_t id, uint32_t resultClassID); - void - assertTensor(const Tensor::UP &exp, - const std::string &fieldName, - const DocsumReply &reply, - uint32_t id, - uint32_t resultClassID); - - bool - assertSlime(const std::string &exp, - const DocsumReply &reply, - uint32_t id, - bool relaxed = false); + bool assertSlime(const std::string &exp, const DocsumReply &reply, uint32_t id,bool relaxed = false); void requireThatAdapterHandlesAllFieldTypes(); void requireThatAdapterHandlesMultipleDocuments(); @@ -346,12 +324,10 @@ GeneralResultPtr Test::getResult(DocumentStoreAdapter & dsa, uint32_t docId) { DocsumStoreValue docsum = dsa.getMappedDocsum(docId); - ASSERT_TRUE(docsum.pt() != NULL); - GeneralResultPtr retval(new GeneralResult(dsa.getResultClass(), - 0, 0, 0)); + ASSERT_TRUE(docsum.pt() != nullptr); + auto retval = std::make_unique<GeneralResult>(dsa.getResultClass()); // skip the 4 byte class id - ASSERT_TRUE(retval->unpack(docsum.pt() + 4, - docsum.len() - 4) == 0); + ASSERT_TRUE(retval->unpack(docsum.pt() + 4, docsum.len() - 4)); return retval; } @@ -359,46 +335,26 @@ Test::getResult(DocumentStoreAdapter & dsa, uint32_t docId) GeneralResultPtr Test::getResult(const DocsumReply & reply, uint32_t id, uint32_t resultClassID) { - GeneralResultPtr retval(new GeneralResult(getResultConfig(). - LookupResultClass(resultClassID), - 0, 0, 0)); + auto retval = std::make_unique<GeneralResult>(getResultConfig().LookupResultClass(resultClassID)); const DocsumReply::Docsum & docsum = reply.docsums[id]; // skip the 4 byte class id - ASSERT_EQUAL(0, retval->unpack(docsum.data.c_str() + 4, docsum.data.size() - 4)); + ASSERT_TRUE(retval->unpack(docsum.data.c_str() + 4, docsum.data.size() - 4)); return retval; } bool Test::assertString(const std::string & exp, const std::string & fieldName, - DocumentStoreAdapter &dsa, - uint32_t id) + DocumentStoreAdapter &dsa, uint32_t id) { GeneralResultPtr res = getResult(dsa, id); - return EXPECT_EQUAL(exp, std::string(res->GetEntry(fieldName.c_str())-> - _stringval, - res->GetEntry(fieldName.c_str())-> - _stringlen)); + return EXPECT_EQUAL(exp, std::string(res->GetEntry(fieldName.c_str())->_stringval, + res->GetEntry(fieldName.c_str())->_stringlen)); } - -bool -Test::assertString(const std::string & exp, const std::string & fieldName, - const DocsumReply & reply, - uint32_t id, uint32_t resultClassID) -{ - GeneralResultPtr res = getResult(reply, id, resultClassID); - return EXPECT_EQUAL(exp, std::string(res->GetEntry(fieldName.c_str())-> - _stringval, - res->GetEntry(fieldName.c_str())-> - _stringlen)); -} - - void Test::assertTensor(const Tensor::UP & exp, const std::string & fieldName, - const DocsumReply & reply, - uint32_t id, uint32_t) + const DocsumReply & reply, uint32_t id, uint32_t) { const DocsumReply::Docsum & docsum = reply.docsums[id]; uint32_t classId; @@ -408,8 +364,7 @@ Test::assertTensor(const Tensor::UP & exp, const std::string & fieldName, vespalib::Slime slime; vespalib::Memory serialized(docsum.data.c_str() + sizeof(classId), docsum.data.size() - sizeof(classId)); - size_t decodeRes = BinaryFormat::decode(serialized, - slime); + size_t decodeRes = BinaryFormat::decode(serialized, slime); ASSERT_EQUAL(decodeRes, serialized.size); EXPECT_EQUAL(exp.get() != nullptr, slime.get()[fieldName].valid()); @@ -547,7 +502,7 @@ Test::requireThatAdapterHandlesMultipleDocuments() } { // doc 2 DocsumStoreValue docsum = dsa.getMappedDocsum(2); - EXPECT_TRUE(docsum.pt() == NULL); + EXPECT_TRUE(docsum.pt() == nullptr); } { // doc 0 (again) GeneralResultPtr res = getResult(dsa, 0); @@ -640,7 +595,7 @@ Test::requireThatDocsumRequestIsProcessed() EXPECT_TRUE(assertSlime("{a:40}", *rep, 1, false)); EXPECT_EQUAL(search::endDocId, rep->docsums[2].docid); EXPECT_EQUAL(gid9, rep->docsums[2].gid); - EXPECT_TRUE(rep->docsums[2].data.get() == NULL); + EXPECT_TRUE(rep->docsums[2].data.get() == nullptr); } @@ -874,11 +829,11 @@ Test::requireThatSummaryAdapterHandlesPutAndRemove() dc._sa->put(1, 1, *exp); IDocumentStore & store = dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore(); Document::UP act = store.read(1, *bc._repo); - EXPECT_TRUE(act.get() != NULL); + EXPECT_TRUE(act.get() != nullptr); EXPECT_EQUAL(exp->getType(), act->getType()); EXPECT_EQUAL("foo", act->getValue("f1")->toString()); dc._sa->remove(2, 1); - EXPECT_TRUE(store.read(1, *bc._repo).get() == NULL); + EXPECT_TRUE(store.read(1, *bc._repo).get() == nullptr); } @@ -928,7 +883,7 @@ Test::requireThatAnnotationsAreUsed() IDocumentStore & store = dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore(); Document::UP act = store.read(1, *bc._repo); - EXPECT_TRUE(act.get() != NULL); + EXPECT_TRUE(act.get() != nullptr); EXPECT_EQUAL(exp->getType(), act->getType()); EXPECT_EQUAL("foo bar", act->getValue("g")->getAsString()); EXPECT_EQUAL("foo bar", act->getValue("dynamicstring")->getAsString()); @@ -1082,7 +1037,7 @@ Test::requireThatUrisAreUsed() IDocumentStore & store = dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore(); Document::UP act = store.read(1, *bc._repo); - EXPECT_TRUE(act.get() != NULL); + EXPECT_TRUE(act.get() != nullptr); EXPECT_EQUAL(exp->getType(), act->getType()); DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0", @@ -1140,7 +1095,7 @@ Test::requireThatPositionsAreUsed() IDocumentStore & store = dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore(); Document::UP act = store.read(1, *bc._repo); - EXPECT_TRUE(act.get() != NULL); + EXPECT_TRUE(act.get() != nullptr); EXPECT_EQUAL(exp->getType(), act->getType()); DocsumRequest req; @@ -1219,7 +1174,7 @@ Test::requireThatRawFieldsWorks() IDocumentStore & store = dc._ddb->getReadySubDB()->getSummaryManager()->getBackingStore(); Document::UP act = store.read(1, *bc._repo); - EXPECT_TRUE(act.get() != NULL); + EXPECT_TRUE(act.get() != nullptr); EXPECT_EQUAL(exp->getType(), act->getType()); DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0", diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp index 64c3455be84..bbd2221f631 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp @@ -3,6 +3,7 @@ #include "document_field_extractor.h" #include <vespa/document/datatype/arraydatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/boolfieldvalue.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/doublefieldvalue.h> @@ -17,6 +18,7 @@ #include <vespa/vespalib/util/exceptions.h> using document::FieldValue; +using document::BoolFieldValue; using document::ByteFieldValue; using document::ShortFieldValue; using document::IntFieldValue; @@ -45,6 +47,7 @@ class SetUndefinedValueVisitor : public FieldValueVisitor { void visit(document::AnnotationReferenceFieldValue &) override { } void visit(ArrayFieldValue &) override { } + void visit(BoolFieldValue &) override { } void visit(ByteFieldValue &value) override { value = getUndefined<int8_t>(); } void visit(Document &) override { } void visit(DoubleFieldValue &value) override { value = getUndefined<double>(); } @@ -65,6 +68,7 @@ class SetUndefinedValueVisitor : public FieldValueVisitor SetUndefinedValueVisitor setUndefinedValueVisitor; const ArrayDataType arrayTypeByte(*DataType::BYTE); +const ArrayDataType arrayTypeBool(*DataType::BOOL); const ArrayDataType arrayTypeShort(*DataType::SHORT); const ArrayDataType arrayTypeInt(*DataType::INT); const ArrayDataType arrayTypeLong(*DataType::LONG); @@ -78,6 +82,8 @@ getArrayType(const DataType &fieldType) switch (fieldType.getId()) { case DataType::Type::T_BYTE: return &arrayTypeByte; + case DataType::Type::T_BOOL: + return &arrayTypeByte; case DataType::Type::T_SHORT: return &arrayTypeShort; case DataType::Type::T_INT: diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp index ea6a16e1547..c65d18a590f 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp @@ -24,9 +24,7 @@ const vespalib::string DOCUMENT_ID_FIELD("documentid"); } bool -DocumentStoreAdapter::writeStringField(const char * buf, - uint32_t buflen, - ResType type) +DocumentStoreAdapter::writeStringField(const char * buf, uint32_t buflen, ResType type) { switch (type) { case RES_STRING: @@ -47,6 +45,8 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type) switch (type) { case RES_BYTE: return _resultPacker.AddByte(value.getAsInt()); + case RES_BOOL: + return _resultPacker.AddByte(value.getAsInt()); case RES_SHORT: return _resultPacker.AddShort(value.getAsInt()); case RES_INT: @@ -63,8 +63,7 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type) case RES_JSONSTRING: { if (value.getClass().inherits(LiteralFieldValueB::classId)) { - const LiteralFieldValueB & lfv = - static_cast<const LiteralFieldValueB &>(value); + auto & lfv = static_cast<const LiteralFieldValueB &>(value); vespalib::stringref s = lfv.getValueRef(); return writeStringField(s.data(), s.size(), type); } else { @@ -86,7 +85,7 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type) { vespalib::nbostream serialized; if (value.getClass().inherits(TensorFieldValue::classId)) { - const TensorFieldValue &tvalue = static_cast<const TensorFieldValue &>(value); + const auto &tvalue = static_cast<const TensorFieldValue &>(value); const std::unique_ptr<Tensor> &tensor = tvalue.getAsTensorPtr(); if (tensor) { vespalib::tensor::TypedBinaryFormat::serialize(serialized, *tensor); @@ -95,9 +94,7 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type) return _resultPacker.AddSerializedTensor(serialized.peek(), serialized.size()); } default: - LOG(warning, - "Unknown docsum field type: %s. Add empty field", - ResultConfig::GetResTypeName(type)); + LOG(warning, "Unknown docsum field type: %s. Add empty field",ResultConfig::GetResTypeName(type)); return _resultPacker.AddEmpty(); } return false; @@ -114,46 +111,32 @@ DocumentStoreAdapter::convertFromSearchDoc(Document &doc, uint32_t docId) if (fieldName == DOCUMENT_ID_FIELD) { StringFieldValue value(doc.getId().toString()); if (!writeField(value, entry->_type)) { - LOG(warning, "Error while writing field '%s' for docId %u", - fieldName.c_str(), docId); + LOG(warning, "Error while writing field '%s' for docId %u", fieldName.c_str(), docId); } continue; } const Field *field = _fieldCache->getField(i); if (!field) { - LOG(debug, - "Did not find field '%s' in the document " - "for docId %u. Adding empty field", + LOG(debug, "Did not find field '%s' in the document for docId %u. Adding empty field", fieldName.c_str(), docId); _resultPacker.AddEmpty(); continue; } FieldValue::UP fieldValue = doc.getValue(*field); - if (fieldValue.get() == NULL) { - LOG(spam, - "No field value for field '%s' in the document " - "for docId %u. Adding empty field", + if ( ! fieldValue) { + LOG(spam, "No field value for field '%s' in the document for docId %u. Adding empty field", fieldName.c_str(), docId); _resultPacker.AddEmpty(); continue; } - LOG(spam, - "writeField(%s): value(%s), type(%d)", - fieldName.c_str(), fieldValue->toString().c_str(), - entry->_type); - FieldValue::UP convertedFieldValue = - SummaryFieldConverter::convertSummaryField(markup, *fieldValue); - if (convertedFieldValue.get() != NULL) { + LOG(spam, "writeField(%s): value(%s), type(%d)", fieldName.c_str(), fieldValue->toString().c_str(), entry->_type); + FieldValue::UP convertedFieldValue = SummaryFieldConverter::convertSummaryField(markup, *fieldValue); + if (convertedFieldValue) { if (!writeField(*convertedFieldValue, entry->_type)) { - LOG(warning, - "Error while writing field '%s' for docId %u", - fieldName.c_str(), docId); + LOG(warning, "Error while writing field '%s' for docId %u", fieldName.c_str(), docId); } } else { - LOG(spam, - "No converted field value for field '%s' " - " in the document " - "for docId %u. Adding empty field", + LOG(spam, "No converted field value for field '%s' in the document for docId %u. Adding empty field", fieldName.c_str(), docId); _resultPacker.AddEmpty(); } @@ -171,46 +154,33 @@ DocumentStoreAdapter(const search::IDocumentStore & docStore, _repo(repo), _resultConfig(resultConfig), _resultClass(resultConfig. - LookupResultClass(resultConfig. - LookupResultClassId(resultClassName. - c_str()))), + LookupResultClass(resultConfig.LookupResultClassId(resultClassName.c_str()))), _resultPacker(&_resultConfig), _fieldCache(fieldCache), _markupFields(markupFields) { } -DocumentStoreAdapter::~DocumentStoreAdapter() {} +DocumentStoreAdapter::~DocumentStoreAdapter() = default; DocsumStoreValue DocumentStoreAdapter::getMappedDocsum(uint32_t docId) { if (!_resultPacker.Init(getSummaryClassId())) { - LOG(warning, - "Error during init of result class '%s' with class id %u", - _resultClass->GetClassName(), getSummaryClassId()); + LOG(warning, "Error during init of result class '%s' with class id %u", _resultClass->GetClassName(), getSummaryClassId()); return DocsumStoreValue(); } Document::UP document = _docStore.read(docId, _repo); - if (document.get() == NULL) { - LOG(debug, - "Did not find summary document for docId %u. " - "Returning empty docsum", - docId); + if ( ! document) { + LOG(debug, "Did not find summary document for docId %u. Returning empty docsum", docId); return DocsumStoreValue(); } - LOG(spam, - "getMappedDocSum(%u): document={\n%s\n}", - docId, - document->toString(true).c_str()); + LOG(spam, "getMappedDocSum(%u): document={\n%s\n}", docId, document->toString(true).c_str()); convertFromSearchDoc(*document, docId); const char * buf; uint32_t buflen; if (!_resultPacker.GetDocsumBlob(&buf, &buflen)) { - LOG(warning, - "Error while getting the docsum blob for docId %u. " - "Returning empty docsum", - docId); + LOG(warning, "Error while getting the docsum blob for docId %u. Returning empty docsum", docId); return DocsumStoreValue(); } return DocsumStoreValue(buf, buflen); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 54961d10fd3..1abe9540859 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -51,7 +51,7 @@ public: SerialNum flushedSerialNum, Time lastFlushTime, searchcorespi::index::IThreadService & summaryService, std::shared_ptr<ICompactableLidSpace> target); - ~ShrinkSummaryLidSpaceFlushTarget(); + ~ShrinkSummaryLidSpaceFlushTarget() override; Task::UP initFlush(SerialNum currentSerial) override; }; @@ -65,7 +65,7 @@ ShrinkSummaryLidSpaceFlushTarget(const vespalib::string &name, Type type, Compon { } -ShrinkSummaryLidSpaceFlushTarget::~ShrinkSummaryLidSpaceFlushTarget() {} +ShrinkSummaryLidSpaceFlushTarget::~ShrinkSummaryLidSpaceFlushTarget() = default; IFlushTarget::Task::UP ShrinkSummaryLidSpaceFlushTarget::initFlush(SerialNum currentSerial) @@ -93,7 +93,7 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, _repo(repo), _markupFields() { - std::unique_ptr<ResultConfig> resultConfig(new ResultConfig()); + auto resultConfig = std::make_unique<ResultConfig>(); if (!resultConfig->ReadConfig(summaryCfg, make_string("SummaryManager(%s)", baseDir.c_str()).c_str())) { std::ostringstream oss; config::OstreamConfigWriter writer(oss); @@ -103,12 +103,11 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, baseDir.c_str(), oss.str().c_str())); } - _juniperConfig.reset(new juniper::Juniper(&_juniperProps, &_wordFolder)); - _docsumWriter.reset(new DynamicDocsumWriter(resultConfig.release(), NULL)); + _juniperConfig = std::make_unique<juniper::Juniper>(&_juniperProps, &_wordFolder); + _docsumWriter = std::make_unique<DynamicDocsumWriter>(resultConfig.release(), nullptr); DynamicDocsumConfig dynCfg(this, _docsumWriter.get()); dynCfg.configure(summarymapCfg); - for (size_t i = 0; i < summarymapCfg.override.size(); ++i) { - const SummarymapConfig::Override & o = summarymapCfg.override[i]; + for (const auto & o : summarymapCfg.override) { if (o.command == "dynamicteaser" || o.command == "textextractor") { vespalib::string markupField = o.arguments; if (markupField.empty()) @@ -118,11 +117,11 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, } } const DocumentType *docType = repo->getDocumentType(docTypeName.getName()); - if (docType != NULL) { - _fieldCacheRepo.reset(new FieldCacheRepo(getResultConfig(), *docType)); + if (docType != nullptr) { + _fieldCacheRepo = std::make_unique<FieldCacheRepo>(getResultConfig(), *docType); } else if (getResultConfig().GetNumResultClasses() == 0) { LOG(debug, "Create empty field cache repo for document type '%s'", docTypeName.toString().c_str()); - _fieldCacheRepo.reset(new FieldCacheRepo()); + _fieldCacheRepo = std::make_unique<FieldCacheRepo>(); } else { throw IllegalArgumentException(make_string("Did not find document type '%s' in current document type repo." " Cannot setup field cache repo for the summary setup", @@ -212,7 +211,7 @@ IFlushTarget::List SummaryManager::getFlushTargets(searchcorespi::index::IThread } void SummaryManager::reconfigure(const LogDocumentStore::Config & config) { - LogDocumentStore & docStore = dynamic_cast<LogDocumentStore &> (*_docStore); + auto & docStore = dynamic_cast<LogDocumentStore &> (*_docStore); docStore.reconfigure(config); } diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp index e689e613886..ca9e331bb62 100644 --- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -286,4 +286,32 @@ TEST("require that MatchData soft_reset retains appropriate state") { EXPECT_EQUAL(new_term->getDocId(), TermFieldMatchData::invalidId()); } +TEST("require that compareWithExactness implements a strict weak ordering") { + TermFieldMatchDataPosition a(0, 1, 100, 1); + TermFieldMatchDataPosition b(0, 2, 100, 1); + TermFieldMatchDataPosition c(0, 2, 100, 1); + TermFieldMatchDataPosition d(0, 3, 100, 3); + TermFieldMatchDataPosition e(0, 3, 100, 3); + TermFieldMatchDataPosition f(0, 4, 100, 1); + + d.setMatchExactness(0.75); + e.setMatchExactness(0.5); + + bool (*cmp)(const TermFieldMatchDataPosition &a, + const TermFieldMatchDataPosition &b) = TermFieldMatchDataPosition::compareWithExactness; + + EXPECT_EQUAL(true, cmp(a, b)); + EXPECT_EQUAL(false, cmp(b, c)); + EXPECT_EQUAL(true, cmp(c, d)); + EXPECT_EQUAL(true, cmp(d, e)); + EXPECT_EQUAL(true, cmp(e, f)); + + EXPECT_EQUAL(false, cmp(b, a)); + EXPECT_EQUAL(false, cmp(c, b)); + EXPECT_EQUAL(false, cmp(d, c)); + EXPECT_EQUAL(false, cmp(e, d)); + EXPECT_EQUAL(false, cmp(f, e)); +} + + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/datastore/CMakeLists.txt b/searchlib/src/vespa/searchlib/datastore/CMakeLists.txt index 0698c57246a..5af7bd21d78 100644 --- a/searchlib/src/vespa/searchlib/datastore/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/datastore/CMakeLists.txt @@ -6,5 +6,6 @@ vespa_add_library(searchlib_datastore OBJECT bufferstate.cpp datastore.cpp datastorebase.cpp + entryref.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/datastore/entryref.cpp b/searchlib/src/vespa/searchlib/datastore/entryref.cpp new file mode 100644 index 00000000000..f7224191d1b --- /dev/null +++ b/searchlib/src/vespa/searchlib/datastore/entryref.cpp @@ -0,0 +1,17 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "entryref.hpp" + +namespace search::datastore { + +template EntryRefT<24u, 8u>::EntryRefT(uint64_t, uint32_t); +template EntryRefT<31u, 1u>::EntryRefT(uint64_t, uint32_t); +template EntryRefT<22u,10u>::EntryRefT(uint64_t, uint32_t); +template EntryRefT<19u,13u>::EntryRefT(uint64_t, uint32_t); +template EntryRefT<18u, 6u>::EntryRefT(uint64_t, uint32_t); +template EntryRefT<15u,17u>::EntryRefT(uint64_t, uint32_t); +template EntryRefT<10u,22u>::EntryRefT(uint64_t, uint32_t); +template EntryRefT<10u,10u>::EntryRefT(uint64_t, uint32_t); +template EntryRefT< 3u, 2u>::EntryRefT(uint64_t, uint32_t); + +} diff --git a/searchlib/src/vespa/searchlib/datastore/entryref.h b/searchlib/src/vespa/searchlib/datastore/entryref.h index f4dec5dbef3..457ffac4e26 100644 --- a/searchlib/src/vespa/searchlib/datastore/entryref.h +++ b/searchlib/src/vespa/searchlib/datastore/entryref.h @@ -3,7 +3,6 @@ #pragma once #include <cstdint> -#include <vespa/vespalib/util/assert.h> namespace search::datastore { @@ -28,12 +27,7 @@ template <uint32_t OffsetBits, uint32_t BufferBits = 32u - OffsetBits> class EntryRefT : public EntryRef { public: EntryRefT() : EntryRef() {} - EntryRefT(uint64_t offset_, uint32_t bufferId_) : - EntryRef((offset_ << BufferBits) + bufferId_) - { - ASSERT_ONCE_OR_LOG(offset_ < offsetSize(), "EntryRefT.offset_overflow", 10000); - ASSERT_ONCE_OR_LOG(bufferId_ < numBuffers(), "EntryRefT.bufferId_overflow", 10000); - } + EntryRefT(uint64_t offset_, uint32_t bufferId_); EntryRefT(const EntryRef & ref_) : EntryRef(ref_.ref()) {} uint32_t hash() const { return offset() + (bufferId() << OffsetBits); } uint64_t offset() const { return _ref >> BufferBits; } diff --git a/searchlib/src/vespa/searchlib/datastore/entryref.hpp b/searchlib/src/vespa/searchlib/datastore/entryref.hpp new file mode 100644 index 00000000000..a7bb9f9b3ef --- /dev/null +++ b/searchlib/src/vespa/searchlib/datastore/entryref.hpp @@ -0,0 +1,18 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "entryref.h" +#include <vespa/vespalib/util/assert.h> + +namespace search::datastore { + +template <uint32_t OffsetBits, uint32_t BufferBits> +EntryRefT<OffsetBits, BufferBits>::EntryRefT(uint64_t offset_, uint32_t bufferId_) : + EntryRef((offset_ << BufferBits) + bufferId_) +{ + ASSERT_ONCE_OR_LOG(offset_ < offsetSize(), "EntryRefT.offset_overflow", 10000); + ASSERT_ONCE_OR_LOG(bufferId_ < numBuffers(), "EntryRefT.bufferId_overflow", 10000); +} + +} diff --git a/searchlib/src/vespa/searchlib/util/sigbushandler.cpp b/searchlib/src/vespa/searchlib/util/sigbushandler.cpp index 9de3f16be67..7356baab131 100644 --- a/searchlib/src/vespa/searchlib/util/sigbushandler.cpp +++ b/searchlib/src/vespa/searchlib/util/sigbushandler.cpp @@ -97,7 +97,7 @@ SigBusHandler::handle(int sig, siginfo_t *si, void *ucv) do { // Protect against multiple threads. TryLockGuard guard; - if (!guard.gotLock()) { + if (!guard.gotLock() || _fired) { raced = true; break; } @@ -121,18 +121,19 @@ SigBusHandler::handle(int sig, siginfo_t *si, void *ucv) sleep(5); return; } - untrap(); // Further bus errors will trigger core dump if (_unwind != nullptr) { // Unit test is using siglongjmp based unwinding sigjmp_buf *unwind = _unwind; _unwind = nullptr; + untrap(); // Further bus errors will trigger core dump siglongjmp(*unwind, 1); } else { // Normal case, sleep 3 seconds (i.e. allow main thread to detect // issue and notify cluster controller) before returning and // likely core dumping. sleep(3); + untrap(); // Further bus errors will trigger core dump } } diff --git a/searchsummary/src/tests/docsumformat/docsum-pack.cpp b/searchsummary/src/tests/docsumformat/docsum-pack.cpp index 7a9834e3fd8..18b38db3fa1 100644 --- a/searchsummary/src/tests/docsumformat/docsum-pack.cpp +++ b/searchsummary/src/tests/docsumformat/docsum-pack.cpp @@ -10,7 +10,6 @@ LOG_SETUP("docsum-pack"); using namespace search::docsummary; - // needed to resolve external symbol from httpd.h on AIX void FastS_block_usr2() {} @@ -20,8 +19,8 @@ class MyApp : public FastOS_Application private: bool _rc; uint32_t _cnt; - search::docsummary::ResultConfig _config; - search::docsummary::ResultPacker _packer; + ResultConfig _config; + ResultPacker _packer; public: MyApp(); @@ -33,33 +32,18 @@ public: { ReportTestResult(line, rc); return rc; } // compare runtime info (,but ignore result class) - bool Equal(search::docsummary::ResEntry *a, search::docsummary::ResEntry *b); - bool Equal(search::docsummary::GeneralResult *a, search::docsummary::GeneralResult *b); - - void TestFieldIndex(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, int idx); - - void TestIntValue(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, uint32_t value); - - void TestDoubleValue(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, double value); + bool Equal(ResEntry *a, ResEntry *b); + bool Equal(GeneralResult *a, GeneralResult *b); - void TestInt64Value(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, uint64_t value); + void TestIntValue(uint32_t line, GeneralResult *gres, const char *field, uint32_t value); + void TestDoubleValue(uint32_t line, GeneralResult *gres, const char *field, double value); + void TestInt64Value(uint32_t line, GeneralResult *gres, const char *field, uint64_t value); + void TestStringValue(uint32_t line, GeneralResult *gres, const char *field, const char *value); + void TestDataValue(uint32_t line, GeneralResult *gres, const char *field, const char *value); - void TestStringValue(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, const char *value); - - void TestDataValue(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, const char *value); - - void TestBasic(); void TestFailLong(); void TestFailShort(); void TestFailOrder(); - void TestCompress(); - void TestCompat(); void TestBasicInplace(); void TestCompressInplace(); @@ -72,7 +56,8 @@ MyApp::MyApp() _config(), _packer(&_config) {} -MyApp::~MyApp() {} + +MyApp::~MyApp() = default; void MyApp::ReportTestResult(uint32_t line, bool rc) @@ -89,7 +74,7 @@ MyApp::ReportTestResult(uint32_t line, bool rc) bool -MyApp::Equal(search::docsummary::ResEntry *a, search::docsummary::ResEntry *b) +MyApp::Equal(ResEntry *a, ResEntry *b) { if (a->_type != b->_type) return false; @@ -106,7 +91,7 @@ MyApp::Equal(search::docsummary::ResEntry *a, search::docsummary::ResEntry *b) bool -MyApp::Equal(search::docsummary::GeneralResult *a, search::docsummary::GeneralResult *b) +MyApp::Equal(GeneralResult *a, GeneralResult *b) { uint32_t numEntries = a->GetClass()->GetNumEntries(); @@ -125,56 +110,36 @@ MyApp::Equal(search::docsummary::GeneralResult *a, search::docsummary::GeneralRe return true; } - -void -MyApp::TestFieldIndex(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, int idx) -{ - bool rc = (gres != NULL && - gres->GetClass()->GetIndexFromName(field) == idx); - - RTR(line, rc); -} - - void -MyApp::TestIntValue(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, uint32_t value) +MyApp::TestIntValue(uint32_t line, GeneralResult *gres, const char *field, uint32_t value) { - search::docsummary::ResEntry *entry - = (gres != NULL) ? gres->GetEntry(field) : NULL; + ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; - bool rc = (entry != NULL && + bool rc = (entry != nullptr && entry->_type == RES_INT && entry->_intval == value); RTR(line, rc); } - void -MyApp::TestDoubleValue(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, double value) +MyApp::TestDoubleValue(uint32_t line, GeneralResult *gres, const char *field, double value) { - search::docsummary::ResEntry *entry - = (gres != NULL) ? gres->GetEntry(field) : NULL; + ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; - bool rc = (entry != NULL && + bool rc = (entry != nullptr && entry->_type == RES_DOUBLE && entry->_doubleval == value); RTR(line, rc); } - void -MyApp::TestInt64Value(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, uint64_t value) +MyApp::TestInt64Value(uint32_t line, GeneralResult *gres, const char *field, uint64_t value) { - search::docsummary::ResEntry *entry - = (gres != NULL) ? gres->GetEntry(field) : NULL; + ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; - bool rc = (entry != NULL && + bool rc = (entry != nullptr && entry->_type == RES_INT64 && entry->_int64val == value); @@ -183,36 +148,29 @@ MyApp::TestInt64Value(uint32_t line, search::docsummary::GeneralResult *gres, void -MyApp::TestStringValue(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, const char *value) +MyApp::TestStringValue(uint32_t line, GeneralResult *gres, const char *field, const char *value) { - search::docsummary::ResEntry *entry - = (gres != NULL) ? gres->GetEntry(field) : NULL; + ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; - bool rc = (entry != NULL && + bool rc = (entry != nullptr && entry->_type == RES_STRING && entry->_stringlen == strlen(value) && strncmp(entry->_stringval, value, entry->_stringlen) == 0); - if (!rc && entry != NULL) { - LOG(warning, - "string value '%.*s' != '%s'", - (int) entry->_stringlen, - entry->_stringval, value); + if (!rc && entry != nullptr) { + LOG(warning,"string value '%.*s' != '%s'", + (int) entry->_stringlen, entry->_stringval, value); } RTR(line, rc); } - void -MyApp::TestDataValue(uint32_t line, search::docsummary::GeneralResult *gres, - const char *field, const char *value) +MyApp::TestDataValue(uint32_t line, GeneralResult *gres, const char *field, const char *value) { - search::docsummary::ResEntry *entry - = (gres != NULL) ? gres->GetEntry(field) : NULL; + ResEntry *entry = (gres != nullptr) ? gres->GetEntry(field) : nullptr; - bool rc = (entry != NULL && + bool rc = (entry != nullptr && entry->_type == RES_DATA && entry->_datalen == strlen(value) && strncmp(entry->_dataval, value, entry->_datalen) == 0); @@ -220,62 +178,6 @@ MyApp::TestDataValue(uint32_t line, search::docsummary::GeneralResult *gres, RTR(line, rc); } - -void -MyApp::TestBasic() -{ - const char *buf; - uint32_t buflen; - - search::docsummary::urlresult *res; - search::docsummary::GeneralResult *gres; - - uint32_t intval = 4; - uint16_t shortval = 2; - uint8_t byteval = 1; - float floatval = 4.5; - double doubleval = 8.75; - uint64_t int64val = 8; - const char *strval = "This is a string"; - const char *datval = "This is data"; - const char *lstrval = "This is a long string"; - const char *ldatval = "This is long data"; - - RTR(__LINE__, _packer.Init(0)); - RTR(__LINE__, _packer.AddInteger(intval)); - RTR(__LINE__, _packer.AddShort(shortval)); - RTR(__LINE__, _packer.AddByte(byteval)); - RTR(__LINE__, _packer.AddFloat(floatval)); - RTR(__LINE__, _packer.AddDouble(doubleval)); - RTR(__LINE__, _packer.AddInt64(int64val)); - RTR(__LINE__, _packer.AddString(strval, strlen(strval))); - RTR(__LINE__, _packer.AddData(datval, strlen(datval))); - RTR(__LINE__, _packer.AddLongString(lstrval, strlen(lstrval))); - RTR(__LINE__, _packer.AddLongData(ldatval, strlen(ldatval))); - RTR(__LINE__, _packer.GetDocsumBlob(&buf, &buflen)); - - res = _config.Unpack(0, 0, 0, buf, buflen); - gres = res->IsGeneral() ? (search::docsummary::GeneralResult *) res : NULL; - - RTR(__LINE__, gres != NULL); - TestIntValue (__LINE__, gres, "integer", 4); - TestIntValue (__LINE__, gres, "short", 2); - TestIntValue (__LINE__, gres, "byte", 1); - TestDoubleValue(__LINE__, gres, "float", floatval); - TestDoubleValue(__LINE__, gres, "double", doubleval); - TestInt64Value (__LINE__, gres, "int64", int64val); - TestStringValue(__LINE__, gres, "string", strval); - TestDataValue (__LINE__, gres, "data", datval); - TestStringValue(__LINE__, gres, "longstring", lstrval); - TestDataValue (__LINE__, gres, "longdata", ldatval); - RTR(__LINE__, (gres != NULL && - gres->GetClass()->GetNumEntries() == 10)); - RTR(__LINE__, (gres != NULL && - gres->GetClass()->GetClassID() == 0)); - delete res; -} - - void MyApp::TestFailLong() { @@ -308,7 +210,6 @@ MyApp::TestFailLong() RTR(__LINE__, !_packer.GetDocsumBlob(&buf, &buflen)); } - void MyApp::TestFailShort() { @@ -371,95 +272,6 @@ MyApp::TestFailOrder() } -void -MyApp::TestCompress() -{ - const char *buf; - uint32_t buflen; - - search::docsummary::urlresult *res; - search::docsummary::GeneralResult *gres; - - const char *lstrval = "string string string"; - const char *ldatval = "data data data"; - - RTR(__LINE__, _packer.Init(2)); - RTR(__LINE__, _packer.AddLongString(lstrval, strlen(lstrval))); - RTR(__LINE__, _packer.AddLongData(ldatval, strlen(ldatval))); - RTR(__LINE__, _packer.GetDocsumBlob(&buf, &buflen)); - - res = _config.Unpack(0, 0, 0, buf, buflen); - gres = res->IsGeneral() ? (search::docsummary::GeneralResult *) res : NULL; - - RTR(__LINE__, gres != NULL); - TestStringValue(__LINE__, gres, "text", lstrval); - TestDataValue (__LINE__, gres, "data", ldatval); - RTR(__LINE__, (gres != NULL && - gres->GetClass()->GetNumEntries() == 2)); - RTR(__LINE__, (gres != NULL && - gres->GetClass()->GetClassID() == 2)); - delete res; -} - - -void -MyApp::TestCompat() -{ - const char *buf; - uint32_t buflen; - - search::docsummary::urlresult *res1; - search::docsummary::GeneralResult *gres1; - - search::docsummary::urlresult *res2; - search::docsummary::GeneralResult *gres2; - - const char *strval = "string string string string"; - const char *datval = "data data data data"; - - RTR(__LINE__, _packer.Init(1)); - RTR(__LINE__, _packer.AddData(strval, strlen(strval))); - RTR(__LINE__, _packer.AddString(datval, strlen(datval))); - RTR(__LINE__, _packer.GetDocsumBlob(&buf, &buflen)); - res1 = _config.Unpack(0, 0, 0, buf, buflen); - gres1 = res1->IsGeneral() ? (search::docsummary::GeneralResult *) res1 : NULL; - - RTR(__LINE__, _packer.Init(2)); - RTR(__LINE__, _packer.AddLongData(strval, strlen(strval))); - RTR(__LINE__, _packer.AddLongString(datval, strlen(datval))); - RTR(__LINE__, _packer.GetDocsumBlob(&buf, &buflen)); - res2 = _config.Unpack(0, 0, 0, buf, buflen); - gres2 = res2->IsGeneral() ? (search::docsummary::GeneralResult *) res2 : NULL; - - RTR(__LINE__, gres1 != NULL); - RTR(__LINE__, gres2 != NULL); - - TestStringValue(__LINE__, gres1, "text", strval); - TestDataValue (__LINE__, gres1, "data", datval); - TestFieldIndex (__LINE__, gres1, "text", 0); - TestFieldIndex (__LINE__, gres1, "data", 1); - RTR(__LINE__, (gres1 != NULL && - gres1->GetClass()->GetNumEntries() == 2)); - - TestStringValue(__LINE__, gres2, "text", strval); - TestDataValue (__LINE__, gres2, "data", datval); - TestFieldIndex (__LINE__, gres2, "text", 0); - TestFieldIndex (__LINE__, gres2, "data", 1); - RTR(__LINE__, (gres2 != NULL && - gres2->GetClass()->GetNumEntries() == 2)); - - RTR(__LINE__, (gres1 != NULL && - gres1->GetClass()->GetClassID() == 1)); - RTR(__LINE__, (gres2 != NULL && - gres2->GetClass()->GetClassID() == 2)); - - RTR(__LINE__, (gres1 != NULL && gres2 != NULL && - Equal(gres1, gres2))); - - delete res1; - delete res2; -} - void MyApp::TestBasicInplace() @@ -467,8 +279,8 @@ MyApp::TestBasicInplace() const char *buf; uint32_t buflen; - const search::docsummary::ResultClass *resClass; - search::docsummary::GeneralResult *gres; + const ResultClass *resClass; + GeneralResult *gres; uint32_t intval = 4; uint16_t shortval = 2; @@ -495,18 +307,18 @@ MyApp::TestBasicInplace() RTR(__LINE__, _packer.GetDocsumBlob(&buf, &buflen)); resClass = _config.LookupResultClass(_config.GetClassID(buf, buflen)); - if (resClass == NULL) { - gres = NULL; + if (resClass == nullptr) { + gres = nullptr; } else { DocsumStoreValue value(buf, buflen); - gres = new search::docsummary::GeneralResult(resClass, 0, 0, 0); + gres = new GeneralResult(resClass); if (!gres->inplaceUnpack(value)) { delete gres; - gres = NULL; + gres = nullptr; } } - RTR(__LINE__, gres != NULL); + RTR(__LINE__, gres != nullptr); TestIntValue (__LINE__, gres, "integer", 4); TestIntValue (__LINE__, gres, "short", 2); TestIntValue (__LINE__, gres, "byte", 1); @@ -517,9 +329,9 @@ MyApp::TestBasicInplace() TestDataValue (__LINE__, gres, "data", datval); TestStringValue(__LINE__, gres, "longstring", lstrval); TestDataValue (__LINE__, gres, "longdata", ldatval); - RTR(__LINE__, (gres != NULL && + RTR(__LINE__, (gres != nullptr && gres->GetClass()->GetNumEntries() == 10)); - RTR(__LINE__, (gres != NULL && + RTR(__LINE__, (gres != nullptr && gres->GetClass()->GetClassID() == 0)); delete gres; } @@ -533,8 +345,8 @@ MyApp::TestCompressInplace() search::RawBuf field1(32768); search::RawBuf field2(32768); - const search::docsummary::ResultClass *resClass; - search::docsummary::GeneralResult *gres; + const ResultClass *resClass; + GeneralResult *gres; const char *lstrval = "string string string"; const char *ldatval = "data data data"; @@ -545,48 +357,46 @@ MyApp::TestCompressInplace() RTR(__LINE__, _packer.GetDocsumBlob(&buf, &buflen)); resClass = _config.LookupResultClass(_config.GetClassID(buf, buflen)); - if (resClass == NULL) { - gres = NULL; + if (resClass == nullptr) { + gres = nullptr; } else { DocsumStoreValue value(buf, buflen); - gres = new search::docsummary::GeneralResult(resClass, 0, 0, 0); + gres = new GeneralResult(resClass); if (!gres->inplaceUnpack(value)) { delete gres; - gres = NULL; + gres = nullptr; } } - search::docsummary::ResEntry *e1 = (gres == NULL) ? NULL : gres->GetEntry("text"); - search::docsummary::ResEntry *e2 = (gres == NULL) ? NULL : gres->GetEntry("data"); + ResEntry *e1 = (gres == nullptr) ? nullptr : gres->GetEntry("text"); + ResEntry *e2 = (gres == nullptr) ? nullptr : gres->GetEntry("data"); - if (e1 != NULL) + if (e1 != nullptr) e1->_extract_field(&field1); - if (e2 != NULL) + if (e2 != nullptr) e2->_extract_field(&field2); - RTR(__LINE__, gres != NULL); - RTR(__LINE__, e1 != NULL); - RTR(__LINE__, e2 != NULL); + RTR(__LINE__, gres != nullptr); + RTR(__LINE__, e1 != nullptr); + RTR(__LINE__, e2 != nullptr); RTR(__LINE__, strcmp(field1.GetDrainPos(), lstrval) == 0); RTR(__LINE__, strcmp(field2.GetDrainPos(), ldatval) == 0); RTR(__LINE__, strlen(lstrval) == field1.GetUsedLen()); RTR(__LINE__, strlen(ldatval) == field2.GetUsedLen()); - RTR(__LINE__, (gres != NULL && + RTR(__LINE__, (gres != nullptr && gres->GetClass()->GetNumEntries() == 2)); - RTR(__LINE__, (gres != NULL && + RTR(__LINE__, (gres != nullptr && gres->GetClass()->GetClassID() == 2)); delete gres; } - - int MyApp::Main() { _rc = true; _cnt = 0; - search::docsummary::ResultClass *resClass; + ResultClass *resClass; resClass = _config.AddResultClass("c0", 0); resClass->AddConfigEntry("integer", RES_INT); @@ -608,12 +418,9 @@ MyApp::Main() resClass->AddConfigEntry("text", RES_LONG_STRING); resClass->AddConfigEntry("data", RES_LONG_DATA); - TestBasic(); TestFailLong(); TestFailShort(); TestFailOrder(); - TestCompress(); - TestCompat(); TestBasicInplace(); TestCompressInplace(); @@ -621,7 +428,6 @@ MyApp::Main() return (_rc ? 0 : 1); } - int main(int argc, char **argv) { diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 7497a66d138..764ff4723cb 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -24,8 +24,7 @@ using search::attribute::IAttributeFunctor; using vespalib::string; using std::vector; -namespace search { -namespace docsummary { +namespace search::docsummary { namespace { @@ -136,7 +135,7 @@ void checkWritePositionField(Test &test, AttrType &attr, vespalib::Slime target; vespalib::slime::SlimeInserter inserter(target); - writer->insertField(doc_id, nullptr, &state, res_type, inserter); + writer->insertField(doc_id, &state, res_type, inserter); vespalib::Memory got = target.get().asString(); test.EXPECT_EQUAL(expected.size(), got.size); @@ -154,7 +153,6 @@ void Test::requireThat2DPositionFieldIsWritten() { } } // namespace -} // namespace docsummary -} // namespace search +} TEST_APPHOOK(search::docsummary::Test); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp index 015eb70c74a..153bcea886f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -90,7 +90,7 @@ StructFields::~StructFields() = default; } AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName) - : IDocsumFieldWriter(), + : IDocsumFW(), _stateIndex(0), _fieldName(fieldName) { @@ -124,11 +124,7 @@ AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManage } void -AttributeCombinerDFW::insertField(uint32_t docid, - GeneralResult *, - GetDocsumsState *state, - ResType, - vespalib::slime::Inserter &target) +AttributeCombinerDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) { auto &fieldWriterState = state->_fieldWriterStates[_stateIndex]; if (!fieldWriterState) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h index ef54522a923..e58b0f740bf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -15,7 +15,7 @@ class DynamicDocsumWriter; * This class reads values from multiple struct field attributes and * inserts them as an array of struct or a map of struct. */ -class AttributeCombinerDFW : public IDocsumFieldWriter +class AttributeCombinerDFW : public IDocsumFW { protected: uint32_t _stateIndex; @@ -28,8 +28,7 @@ public: bool IsGenerated() const override; bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, IAttributeManager &attrMgr); - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) override; + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index ee2e9d9fa92..19dd096c46c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -21,6 +21,7 @@ using namespace search; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; using search::attribute::BasicType; +using vespalib::slime::Inserter; namespace search::docsummary { @@ -42,12 +43,8 @@ public: SingleAttrDFW(const vespalib::string & attrName) : AttrDFW(attrName) { } - virtual void insertField(uint32_t docid, - GeneralResult *gres, - GetDocsumsState *state, - ResType type, - vespalib::slime::Inserter &target) override; - virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const override; + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, Inserter &target) override; + bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const override; }; bool SingleAttrDFW::isDefaultValue(uint32_t docid, const GetDocsumsState * state) const @@ -56,11 +53,7 @@ bool SingleAttrDFW::isDefaultValue(uint32_t docid, const GetDocsumsState * state } void -SingleAttrDFW::insertField(uint32_t docid, - GeneralResult *, - GetDocsumsState * state, - ResType type, - vespalib::slime::Inserter &target) +SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type, Inserter &target) { const char *s=""; const IAttributeVector & v = vec(*state); @@ -80,6 +73,11 @@ SingleAttrDFW::insertField(uint32_t docid, target.insertLong(val); break; } + case RES_BOOL: { + uint8_t val = v.getInt(docid); + target.insertBool(val != 0); + break; + } case RES_FLOAT: { float val = v.getFloat(docid); target.insertDouble(val); @@ -141,20 +139,12 @@ class MultiAttrDFW : public AttrDFW { public: MultiAttrDFW(const vespalib::string & attrName) : AttrDFW(attrName) {} - virtual void insertField(uint32_t docid, - GeneralResult *gres, - GetDocsumsState *state, - ResType type, - vespalib::slime::Inserter &target) override; + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, Inserter &target) override; }; void -MultiAttrDFW::insertField(uint32_t docid, - GeneralResult *, - GetDocsumsState *state, - ResType, - vespalib::slime::Inserter &target) +MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, Inserter &target) { using vespalib::slime::Cursor; using vespalib::Memory; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h index 643170663b8..d8102734452 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h @@ -8,7 +8,7 @@ namespace search::attribute { class IAttributeVector; } namespace search::docsummary { -class AttrDFW : public IDocsumFieldWriter +class AttrDFW : public IDocsumFW { private: vespalib::string _attrName; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp index 18e7e471663..f24f5841bd2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp @@ -29,22 +29,16 @@ IDocsumFieldWriter::setFieldWriterStateIndex(uint32_t) //-------------------------------------------------------------------------- -EmptyDFW::EmptyDFW() { } +EmptyDFW::EmptyDFW() = default; - -EmptyDFW::~EmptyDFW() { } +EmptyDFW::~EmptyDFW() = default; void -EmptyDFW::insertField(uint32_t /*docid*/, - GeneralResult *, - GetDocsumsState *, - ResType, - vespalib::slime::Inserter &target) +EmptyDFW::insertField(uint32_t, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) { // insert explicitly-empty field? // target.insertNix(); (void)target; - return; } //-------------------------------------------------------------------------- @@ -54,11 +48,7 @@ CopyDFW::CopyDFW() { } - -CopyDFW::~CopyDFW() -{ -} - +CopyDFW::~CopyDFW() = default; bool CopyDFW::Init(const ResultConfig & config, const char *inputField) @@ -69,11 +59,10 @@ CopyDFW::Init(const ResultConfig & config, const char *inputField) LOG(warning, "no docsum format contains field '%s'; copied fields will be empty", inputField); } - for (ResultConfig::const_iterator it(config.begin()), mt(config.end()); it != mt; it++) { - const ResConfigEntry *entry = - it->GetEntry(it->GetIndexFromEnumValue(_inputFieldEnumValue)); + for (const auto & field : config) { + const ResConfigEntry *entry = field.GetEntry(field.GetIndexFromEnumValue(_inputFieldEnumValue)); - if (entry != NULL && + if (entry != nullptr && !IsRuntimeCompatible(entry->_type, RES_INT) && !IsRuntimeCompatible(entry->_type, RES_DOUBLE) && !IsRuntimeCompatible(entry->_type, RES_INT64) && @@ -81,25 +70,21 @@ CopyDFW::Init(const ResultConfig & config, const char *inputField) !IsRuntimeCompatible(entry->_type, RES_DATA)) { LOG(warning, "cannot use docsum field '%s' as input to copy; type conflict with result class %d (%s)", - inputField, it->GetClassID(), it->GetClassName()); + inputField, field.GetClassID(), field.GetClassName()); return false; } } return true; } - void -CopyDFW::insertField(uint32_t /*docid*/, - GeneralResult *gres, - GetDocsumsState *state, - ResType type, - vespalib::slime::Inserter &target) +CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *state, ResType type, + vespalib::slime::Inserter &target) { int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); ResEntry *entry = gres->GetEntry(idx); - if (entry != NULL && + if (entry != nullptr && IsRuntimeCompatible(entry->_type, type)) { switch (type) { @@ -117,6 +102,9 @@ CopyDFW::insertField(uint32_t /*docid*/, uint8_t val8 = entry->_intval; target.insertLong(val8); break; } + case RES_BOOL: { + target.insertBool(entry->_intval != 0); + break; } case RES_FLOAT: { float valfloat = entry->_doubleval; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h index 51079f7736e..870e4fa8d83 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h @@ -16,7 +16,7 @@ class GetDocsumsState; class IDocsumFieldWriter { public: - typedef std::unique_ptr<IDocsumFieldWriter> UP; + using UP = std::unique_ptr<IDocsumFieldWriter>; IDocsumFieldWriter() : _index(0) { } virtual ~IDocsumFieldWriter() {} @@ -27,10 +27,7 @@ public: { return ResultConfig::IsRuntimeCompatible(a, b); } virtual bool IsGenerated() const = 0; - virtual void insertField(uint32_t docid, - GeneralResult *gres, - GetDocsumsState *state, - ResType type, + virtual void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; virtual const vespalib::string & getAttributeName() const { return _empty; } virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const { @@ -46,20 +43,27 @@ private: static const vespalib::string _empty; }; +class IDocsumFW : public IDocsumFieldWriter +{ +public: + virtual void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) = 0; + void insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, ResType type, + vespalib::slime::Inserter &target) override + { + insertField(docid, state, type, target); + } +}; + //-------------------------------------------------------------------------- -class EmptyDFW : public IDocsumFieldWriter +class EmptyDFW : public IDocsumFW { public: EmptyDFW(); - virtual ~EmptyDFW(); - - virtual bool IsGenerated() const override { return true; } - virtual void insertField(uint32_t docid, - GeneralResult *gres, - GetDocsumsState *state, - ResType type, - vespalib::slime::Inserter &target) override; + ~EmptyDFW() override; + + bool IsGenerated() const override { return true; } + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; //-------------------------------------------------------------------------- @@ -71,16 +75,13 @@ private: public: CopyDFW(); - virtual ~CopyDFW(); + ~CopyDFW() override; bool Init(const ResultConfig & config, const char *inputField); - virtual bool IsGenerated() const override { return false; } - virtual void insertField(uint32_t docid, - GeneralResult *gres, - GetDocsumsState *state, - ResType type, - vespalib::slime::Inserter &target) override; + bool IsGenerated() const override { return false; } + void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, + vespalib::slime::Inserter &target) override; }; //-------------------------------------------------------------------------- diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 722ea9d9000..a188ab6e60a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -78,23 +78,23 @@ DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, uint32_t id) const } } -static void convertEntry(GetDocsumsState *state, - const ResConfigEntry *resCfg, - const ResEntry *entry, - Inserter &inserter, - Slime &slime) +static void convertEntry(GetDocsumsState *state, const ResConfigEntry *resCfg, const ResEntry *entry, + Inserter &inserter, Slime &slime) { using vespalib::slime::BinaryFormat; const char *ptr; uint32_t len; - LOG_ASSERT(resCfg != 0 && entry != 0); + LOG_ASSERT(resCfg != nullptr && entry != nullptr); switch (resCfg->_type) { case RES_INT: case RES_SHORT: case RES_BYTE: inserter.insertLong(entry->_intval); break; + case RES_BOOL: + inserter.insertBool(entry->_intval != 0); + break; case RES_FLOAT: case RES_DOUBLE: inserter.insertDouble(entry->_doubleval); @@ -130,12 +130,8 @@ static void convertEntry(GetDocsumsState *state, void -DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, - uint32_t docid, - GetDocsumsState *state, - IDocsumStore *docinfos, - vespalib::Slime & slime, - vespalib::slime::Inserter & topInserter) +DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, + IDocsumStore *docinfos, vespalib::Slime & slime, vespalib::slime::Inserter & topInserter) { if (rci.allGenerated) { // generate docsum entry on-the-fly @@ -144,8 +140,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, const ResConfigEntry *resCfg = rci.outputClass->GetEntry(i); IDocsumFieldWriter *writer = _overrideTable[resCfg->_enumValue]; if (! writer->isDefaultValue(docid, state)) { - const Memory field_name(resCfg->_bindname.data(), - resCfg->_bindname.size()); + const Memory field_name(resCfg->_bindname.data(), resCfg->_bindname.size()); ObjectInserter inserter(docsum, field_name); writer->insertField(docid, nullptr, state, resCfg->_type, inserter); } @@ -154,7 +149,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, // look up docsum entry DocsumStoreValue value = docinfos->getMappedDocsum(docid); // re-pack docsum blob - GeneralResult gres(rci.inputClass, 0, docid, 0); + GeneralResult gres(rci.inputClass); if (! gres.inplaceUnpack(value)) { LOG(debug, "Unpack failed: illegal docsum entry for document %d. This is expected during lidspace compaction.", docid); topInserter.insertNix(); @@ -272,10 +267,10 @@ DynamicDocsumWriter::Override(const char *fieldName, IDocsumFieldWriter *writer) ++_numFieldWriterStates; } - for (ResultConfig::iterator it(_resultConfig->begin()), mt(_resultConfig->end()); it != mt; it++) { + for (auto & entry : *_resultConfig) { - if (it->GetIndexFromEnumValue(fieldEnumValue) >= 0) { - ResultClass::DynamicInfo *info = it->getDynamicInfo(); + if (entry.GetIndexFromEnumValue(fieldEnumValue) >= 0) { + ResultClass::DynamicInfo *info = entry.getDynamicInfo(); info->_overrideCnt++; if (writer->IsGenerated()) info->_generateCnt++; @@ -306,15 +301,11 @@ DynamicDocsumWriter::InitState(IAttributeManager & attrMan, GetDocsumsState *sta uint32_t -DynamicDocsumWriter::WriteDocsum(uint32_t docid, - GetDocsumsState *state, - IDocsumStore *docinfos, - search::RawBuf *target) +DynamicDocsumWriter::WriteDocsum(uint32_t docid, GetDocsumsState *state, IDocsumStore *docinfos, search::RawBuf *target) { vespalib::Slime slime; vespalib::slime::SlimeInserter inserter(slime); - ResolveClassInfo rci = resolveClassInfo(state->_args.getResultClassName(), - docinfos->getSummaryClassId()); + ResolveClassInfo rci = resolveClassInfo(state->_args.getResultClassName(), docinfos->getSummaryClassId()); insertDocsum(rci, docid, state, docinfos, slime, inserter); return slime2RawBuf(slime, *target); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 4112afc1895..b9177ac8782 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -23,7 +23,7 @@ struct ExplicitItemData uint32_t _weight; ExplicitItemData() - : _index(NULL), _indexlen(0), _term(NULL), _termlen(0), _weight(0) + : _index(nullptr), _indexlen(0), _term(nullptr), _termlen(0), _weight(0) {} ExplicitItemData(const char *index, uint32_t indexlen, const char* term, @@ -43,19 +43,16 @@ struct QueryItem { search::SimpleQueryStackDumpIterator *_si; const ExplicitItemData *_data; - QueryItem() : _si(NULL), _data(NULL) {} - QueryItem(search::SimpleQueryStackDumpIterator *si) : _si(si), _data(NULL) {} - QueryItem(ExplicitItemData *data) : _si(NULL), _data(data) {} + QueryItem() : _si(nullptr), _data(nullptr) {} + QueryItem(search::SimpleQueryStackDumpIterator *si) : _si(si), _data(nullptr) {} + QueryItem(ExplicitItemData *data) : _si(nullptr), _data(data) {} private: QueryItem(const QueryItem&); QueryItem& operator= (const QueryItem&); }; -}; - -namespace search { -class Property; +} -namespace fef { +namespace search::fef { class TermVisitor : public IPropertiesVisitor { public: @@ -104,24 +101,22 @@ TermVisitor::visitProperty(const Property::Value &key, const Property &values) } -namespace docsummary { +namespace search::docsummary { class JuniperQueryAdapter : public juniper::IQuery { private: - JuniperQueryAdapter(const JuniperQueryAdapter&); - JuniperQueryAdapter operator= (const JuniperQueryAdapter&); - KeywordExtractor *_kwExtractor; const vespalib::stringref _buf; const search::fef::Properties *_highlightTerms; juniper::IQueryVisitor *_visitor; public: - JuniperQueryAdapter(KeywordExtractor *kwExtractor, - vespalib::stringref buf, - const search::fef::Properties *highlightTerms = NULL) - : _kwExtractor(kwExtractor), _buf(buf), _highlightTerms(highlightTerms), _visitor(NULL) {} + JuniperQueryAdapter(const JuniperQueryAdapter&) = delete; + JuniperQueryAdapter operator= (const JuniperQueryAdapter&) = delete; + JuniperQueryAdapter(KeywordExtractor *kwExtractor, vespalib::stringref buf, + const search::fef::Properties *highlightTerms = nullptr) + : _kwExtractor(kwExtractor), _buf(buf), _highlightTerms(highlightTerms), _visitor(nullptr) {} // TODO: put this functionality into the stack dump iterator bool SkipItem(search::SimpleQueryStackDumpIterator *iterator) const @@ -136,28 +131,28 @@ public: return true; } - virtual bool Traverse(juniper::IQueryVisitor *v) const override; + bool Traverse(juniper::IQueryVisitor *v) const override; - virtual int Weight(const juniper::QueryItem* item) const override + int Weight(const juniper::QueryItem* item) const override { - if (item->_si != NULL) { + if (item->_si != nullptr) { return item->_si->GetWeight().percent(); } else { return item->_data->_weight; } } - virtual juniper::ItemCreator Creator(const juniper::QueryItem* item) const override + juniper::ItemCreator Creator(const juniper::QueryItem* item) const override { // cast master: Knut Omang - if (item->_si != NULL) { + if (item->_si != nullptr) { return (juniper::ItemCreator) item->_si->getCreator(); } else { return juniper::CREA_ORIG; } } - virtual const char *Index(const juniper::QueryItem* item, size_t *len) const override + const char *Index(const juniper::QueryItem* item, size_t *len) const override { - if (item->_si != NULL) { + if (item->_si != nullptr) { *len = item->_si->getIndexName().size(); return item->_si->getIndexName().data(); } else { @@ -166,14 +161,14 @@ public: } } - virtual bool UsefulIndex(const juniper::QueryItem* item) const override + bool UsefulIndex(const juniper::QueryItem* item) const override { vespalib::stringref index; - if (_kwExtractor == NULL) + if (_kwExtractor == nullptr) return true; - if (item->_si != NULL) { + if (item->_si != nullptr) { index = item->_si->getIndexName(); } else { index = vespalib::stringref(item->_data->_index, item->_data->_indexlen); @@ -182,8 +177,6 @@ public: } }; - - bool JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const { @@ -308,7 +301,7 @@ JuniperDFW::JuniperDFW(juniper::Juniper * juniper) } -JuniperDFW::~JuniperDFW() { } +JuniperDFW::~JuniperDFW() = default; bool JuniperDFW::Init( @@ -319,10 +312,10 @@ JuniperDFW::Init( { bool rc = true; const util::StringEnum & enums(config.GetFieldNameEnum()); - if (langFieldName != NULL) + if (langFieldName != nullptr) _langFieldEnumValue = enums.Lookup(langFieldName); _juniperConfig = _juniper->CreateConfig(fieldName); - if (_juniperConfig.get() == NULL) { + if (_juniperConfig.get() == nullptr) { LOG(warning, "could not create juniper config for field '%s'", fieldName); rc = false; } @@ -350,7 +343,7 @@ JuniperTeaserDFW::Init( const ResConfigEntry *entry = it->GetEntry(it->GetIndexFromEnumValue(_inputFieldEnumValue)); - if (entry != NULL && + if (entry != nullptr && !IsRuntimeCompatible(entry->_type, RES_STRING) && !IsRuntimeCompatible(entry->_type, RES_DATA)) { @@ -363,15 +356,13 @@ JuniperTeaserDFW::Init( } vespalib::string -DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, - GeneralResult *gres, - GetDocsumsState *state) +DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, GeneralResult *gres, GetDocsumsState *state) { - if (state->_dynteaser._query == NULL) { + if (state->_dynteaser._query == nullptr) { JuniperQueryAdapter iq(state->_kwExtractor, state->_args.getStackDump(), &state->_args.highlightTerms()); - state->_dynteaser._query = _juniper->CreateQueryHandle(iq, NULL); + state->_dynteaser._query = _juniper->CreateQueryHandle(iq, nullptr); } if (docid != state->_dynteaser._docid || @@ -384,34 +375,31 @@ DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, _langFieldEnumValue, state->_dynteaser._lang, (juniper::AnalyseCompatible(_juniperConfig.get(), state->_dynteaser._config) ? "no" : "yes")); - if (state->_dynteaser._result != NULL) + if (state->_dynteaser._result != nullptr) juniper::ReleaseResult(state->_dynteaser._result); state->_dynteaser._docid = docid; state->_dynteaser._input = _inputFieldEnumValue; state->_dynteaser._lang = _langFieldEnumValue; state->_dynteaser._config = _juniperConfig.get(); - state->_dynteaser._result = NULL; + state->_dynteaser._result = nullptr; int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue); ResEntry *entry = gres->GetEntry(idx); - if (entry != NULL && - state->_dynteaser._query != NULL) { + if (entry != nullptr && state->_dynteaser._query != nullptr) { // obtain Juniper input const char *buf; uint32_t buflen; - entry->_resolve_field(&buf, &buflen, - &state->_docSumFieldSpace); + entry->_resolve_field(&buf, &buflen, &state->_docSumFieldSpace); if (LOG_WOULD_LOG(spam)) { std::ostringstream hexDump; hexDump << vespalib::HexDump(buf, buflen); LOG(spam, "makeDynamicTeaser: docid=%d, input='%s', hexdump:\n%s", - docid, std::string(buf, buflen).c_str(), - hexDump.str().c_str()); + docid, std::string(buf, buflen).c_str(), hexDump.str().c_str()); } uint32_t langid = static_cast<uint32_t>(-1); @@ -422,33 +410,29 @@ DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, } } - juniper::Summary *teaser = (state->_dynteaser._result != NULL) + juniper::Summary *teaser = (state->_dynteaser._result != nullptr) ? juniper::GetTeaser(state->_dynteaser._result, _juniperConfig.get()) - : NULL; + : nullptr; if (LOG_WOULD_LOG(debug)) { std::ostringstream hexDump; - if (teaser != NULL) { + if (teaser != nullptr) { hexDump << vespalib::HexDump(teaser->Text(), teaser->Length()); } LOG(debug, "makeDynamicTeaser: docid=%d, teaser='%s', hexdump:\n%s", - docid, (teaser != NULL ? std::string(teaser->Text(), teaser->Length()).c_str() : "NULL"), + docid, (teaser != nullptr ? std::string(teaser->Text(), teaser->Length()).c_str() : "nullptr"), hexDump.str().c_str()); } - if (teaser != NULL) { - return vespalib::string(teaser->Text(), - teaser->Length()); + if (teaser != nullptr) { + return vespalib::string(teaser->Text(), teaser->Length()); } else { return vespalib::string(); } } void -DynamicTeaserDFW::insertField(uint32_t docid, - GeneralResult *gres, - GetDocsumsState *state, - ResType, +DynamicTeaserDFW::insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) { vespalib::string teaser = makeDynamicTeaser(docid, gres, state); @@ -456,6 +440,4 @@ DynamicTeaserDFW::insertField(uint32_t docid, target.insertString(value); } -} // namespace docsummary -} // namespace search - +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index bf010172fa9..ae3d6acde43 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -40,8 +40,7 @@ void fmtZcurve(int64_t zval, vespalib::slime::Inserter &target) } void -GeoPositionDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState * dsState, - ResType, vespalib::slime::Inserter &target) +GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, vespalib::slime::Inserter &target) { using vespalib::slime::Cursor; using vespalib::slime::ObjectSymbolInserter; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h index 8f630cde3af..9bd85abaf17 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h @@ -14,8 +14,7 @@ class GeoPositionDFW : public AttrDFW public: typedef std::unique_ptr<GeoPositionDFW> UP; GeoPositionDFW(const vespalib::string & attrName); - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) override; + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; static UP create(const char *attribute_name, IAttributeManager *attribute_manager); }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 48e79a5e34c..9c82c00c3ef 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -66,8 +66,7 @@ AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state) } void -AbsDistanceDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) +AbsDistanceDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) { bool forceEmpty = true; @@ -166,8 +165,7 @@ formatField(const attribute::IAttributeVector &attribute, uint32_t docid, ResTyp } void -PositionsDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState * dsState, - ResType type, vespalib::slime::Inserter &target) +PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType type, vespalib::slime::Inserter &target) { vespalib::asciistream val(formatField(vec(*dsState), docid, type)); target.insertString(vespalib::Memory(val.c_str(), val.size())); @@ -175,8 +173,7 @@ PositionsDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState * dsS //-------------------------------------------------------------------------- -PositionsDFW::UP createPositionsDFW(const char *attribute_name, - IAttributeManager *attribute_manager) +PositionsDFW::UP createPositionsDFW(const char *attribute_name, IAttributeManager *attribute_manager) { PositionsDFW::UP ret; if (attribute_manager != nullptr) { @@ -195,12 +192,10 @@ PositionsDFW::UP createPositionsDFW(const char *attribute_name, return ret; } } - ret.reset(new PositionsDFW(attribute_name)); - return ret; + return std::make_unique<PositionsDFW>(attribute_name); } -AbsDistanceDFW::UP createAbsDistanceDFW(const char *attribute_name, - IAttributeManager *attribute_manager) +AbsDistanceDFW::UP createAbsDistanceDFW(const char *attribute_name, IAttributeManager *attribute_manager) { AbsDistanceDFW::UP ret; if (attribute_manager != nullptr) { @@ -219,8 +214,7 @@ AbsDistanceDFW::UP createAbsDistanceDFW(const char *attribute_name, return ret; } } - ret.reset(new AbsDistanceDFW(attribute_name)); - return ret; + return std::make_unique<AbsDistanceDFW>(attribute_name); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h index 69a7ba3f58f..999da6f1860 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h @@ -14,7 +14,7 @@ public: AbsDistanceDFW(const vespalib::string & attrName); bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; @@ -28,8 +28,7 @@ public: PositionsDFW(const vespalib::string & attrName); bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) override ; + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; PositionsDFW::UP createPositionsDFW(const char *attribute_name, IAttributeManager *index_man); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index 9748bdac3b3..a1c96bb3e5b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -20,7 +20,7 @@ RankFeaturesDFW::init(IDocsumEnvironment * env) } void -RankFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, +RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) { if (state->_rankFeatures.get() == nullptr) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h index 04ee14c79ca..37790d2f9b8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h @@ -19,8 +19,7 @@ public: ~RankFeaturesDFW(); void init(IDocsumEnvironment * env); bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) override; + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index e7c7c799b5f..52e331cd365 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -18,6 +18,7 @@ namespace search::docsummary { enum ResType { RES_INT = 0, RES_SHORT, + RES_BOOL, RES_BYTE, RES_FLOAT, RES_DOUBLE, diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index 1c42709826f..f3834ef4a12 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -45,6 +45,7 @@ ResultConfig::GetResTypeName(ResType type) case RES_INT: return "integer"; case RES_SHORT: return "short"; case RES_BYTE: return "byte"; + case RES_BOOL: return "bool"; case RES_FLOAT: return "float"; case RES_DOUBLE: return "double"; case RES_INT64: return "int64"; @@ -73,15 +74,14 @@ ResultConfig::Reset() ResultClass * ResultConfig::AddResultClass(const char *name, uint32_t id) { - ResultClass *ret = NULL; + ResultClass *ret = nullptr; if (id != NoClassID() && (_classLookup.find(id) == _classLookup.end())) { ResultClass::UP rc(new ResultClass(name, id, _fieldEnum)); ret = rc.get(); _classLookup[id] = std::move(rc); if (_nameLookup.find(name) != _nameLookup.end()) { - LOG(warning, "Duplicate result class name: %s " - "(now maps to class id %u)", name, id); + LOG(warning, "Duplicate result class name: %s (now maps to class id %u)", name, id); } _nameLookup[name] = id; } @@ -93,7 +93,7 @@ const ResultClass* ResultConfig::LookupResultClass(uint32_t id) const { IdMap::const_iterator it(_classLookup.find(id)); - return (it != _classLookup.end()) ? it->second.get() : NULL; + return (it != _classLookup.end()) ? it->second.get() : nullptr; } uint32_t @@ -113,8 +113,8 @@ ResultConfig::LookupResultClassId(const vespalib::string &name) const void ResultConfig::CreateEnumMaps() { - for (IdMap::iterator it(_classLookup.begin()), mt(_classLookup.end()); it != mt; it++) { - it ->second->CreateEnumMap(); + for (auto & entry : _classLookup) { + entry.second->CreateEnumMap(); } } @@ -137,14 +137,12 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const break; } ResultClass *resClass = AddResultClass(cfg.classes[i].name.c_str(), classID); - if (resClass == NULL) { - LOG(error, - "%s: unable to add classes[%d] name %s", - configId, i, cfg.classes[i].name.c_str()); + if (resClass == nullptr) { + LOG(error,"%s: unable to add classes[%d] name %s", configId, i, cfg.classes[i].name.c_str()); rc = false; break; } - for (unsigned int j = 0; rc && j < cfg.classes[i].fields.size(); j++) { + for (unsigned int j = 0; rc && (j < cfg.classes[i].fields.size()); j++) { const char *fieldtype = cfg.classes[i].fields[j].type.c_str(); const char *fieldname = cfg.classes[i].fields[j].name.c_str(); LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg.classes[i].name.c_str(), fieldname, fieldtype); @@ -152,6 +150,8 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const rc = resClass->AddConfigEntry(fieldname, RES_INT); } else if (strcmp(fieldtype, "short") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_SHORT); + } else if (strcmp(fieldtype, "bool") == 0) { + rc = resClass->AddConfigEntry(fieldname, RES_BOOL); } else if (strcmp(fieldtype, "byte") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_BYTE); } else if (strcmp(fieldtype, "float") == 0) { @@ -176,17 +176,13 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const rc = resClass->AddConfigEntry(fieldname, RES_TENSOR); } else if (strcmp(fieldtype, "featuredata") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_FEATUREDATA); - } else { // FAIL: unknown field type - LOG(error, - "%s %s.fields[%d]: unknown type '%s'", - configId, cfg.classes[i].name.c_str(), j, fieldtype); + } else { + LOG(error, "%s %s.fields[%d]: unknown type '%s'", configId, cfg.classes[i].name.c_str(), j, fieldtype); rc = false; break; } - if (!rc) { // FAIL: duplicate field name - LOG(error, - "%s %s.fields[%d]: duplicate name '%s'", - configId, cfg.classes[i].name.c_str(), j, fieldname); + if (!rc) { + LOG(error, "%s %s.fields[%d]: duplicate name '%s'", configId, cfg.classes[i].name.c_str(), j, fieldname); break; } } @@ -212,33 +208,5 @@ ResultConfig::GetClassID(const char *buf, uint32_t buflen) return ret; } -urlresult* -ResultConfig::Unpack(uint32_t partition, - uint32_t docid, - HitRank metric, - const char *buf, - uint32_t buflen) const -{ - urlresult *ret = NULL; - const ResultClass *resClass = NULL; - uint32_t tmp32; - - if (buflen >= sizeof(tmp32)) { - memcpy(&tmp32, buf, sizeof(tmp32)); - buf += sizeof(tmp32); - buflen -= sizeof(tmp32); - resClass = LookupResultClass(tmp32); - } - - if (resClass != NULL && (buflen > 0)) { - ret = new GeneralResult(resClass, partition, docid, metric); - if (ret->unpack(buf, buflen) != 0) { // FAIL: unpack - delete ret; - ret = NULL; - } - } - - return (ret != NULL) ? ret : new badurlresult(partition, docid, metric); -} } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h index eac6d4b113f..31218d94e93 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h @@ -115,6 +115,9 @@ public: return true; } switch (a) { + case RES_BYTE: + case RES_BOOL: + return (b == RES_BYTE || b == RES_BOOL); case RES_STRING: case RES_DATA: return (b == RES_STRING || b == RES_DATA); @@ -147,7 +150,8 @@ public: case RES_INT: case RES_SHORT: case RES_BYTE: - return (b == RES_INT || b == RES_SHORT || b == RES_BYTE); + case RES_BOOL: + return (b == RES_INT || b == RES_SHORT || b == RES_BYTE || b == RES_BOOL); case RES_FLOAT: case RES_DOUBLE: return (b == RES_FLOAT || b == RES_DOUBLE); @@ -269,29 +273,6 @@ public: * @param buflen length of docsum blob. **/ uint32_t GetClassID(const char *buf, uint32_t buflen); - - /** - * Unpack docsum blob. The first n (0/8/16/32) bits are read from - * the data given and used to look up the appropriate result - * class. A GeneralResult object is created based on that - * class and told to unpack the rest of the docsum blob. If this - * operation succeeds, the GeneralResult object is - * returned. It if fails, a badurlresult object is returned - * instead. - * - * @return object representing the unpacked result. - * @param partition partition path for current hit. - * @param docid docid for current hit. - * @param metric relevance estimate for current hit. - * @param buf docsum blob. - * @param buflen length of docsum blob. - **/ - urlresult * - Unpack(uint32_t partition, - uint32_t docid, - HitRank metric, - const char *buf, - uint32_t buflen) const; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp index 178e1a90667..c9642b80e56 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp @@ -23,7 +23,7 @@ bool ResultPacker::CheckEntry(ResType type) if (_error) return false; - bool rc = (_cfgEntry != NULL && + bool rc = (_cfgEntry != nullptr && IsBinaryCompatible(_cfgEntry->_type, type)); if (rc) { @@ -43,7 +43,7 @@ ResultPacker::SetFormatError(ResType type) { _error = true; - if (_cfgEntry != NULL) { + if (_cfgEntry != nullptr) { LOG(error, "ResultPacker: format error: got '%s', expected '%s'", GetResTypeName(type), @@ -60,17 +60,15 @@ ResultPacker::ResultPacker(const ResultConfig *resConfig) : _buf(32768), _cbuf(32768), _resConfig(resConfig), - _resClass(NULL), + _resClass(nullptr), _entryIdx(0), - _cfgEntry(NULL), + _cfgEntry(nullptr), _error(true) { } -ResultPacker::~ResultPacker() -{ -} +ResultPacker::~ResultPacker() = default; void ResultPacker::InitPlain() @@ -82,16 +80,16 @@ bool ResultPacker::Init(uint32_t classID) { _buf.reset(); - _resClass = (_resConfig != NULL) ? - _resConfig->LookupResultClass(classID) : NULL; + _resClass = (_resConfig != nullptr) ? + _resConfig->LookupResultClass(classID) : nullptr; _entryIdx = 0; - if (_resClass != NULL) { + if (_resClass != nullptr) { uint32_t id = _resClass->GetClassID(); _buf.append(&id, sizeof(id)); _cfgEntry = _resClass->GetEntry(_entryIdx); _error = false; } else { - _cfgEntry = NULL; + _cfgEntry = nullptr; _error = true; LOG(error, "ResultPacker: resultclass %d does not exist", classID); @@ -104,22 +102,23 @@ ResultPacker::Init(uint32_t classID) bool ResultPacker::AddEmpty() { - if (!_error && _cfgEntry != NULL) { + if (!_error && _cfgEntry != nullptr) { switch (_cfgEntry->_type) { case RES_INT: return AddInteger(search::attribute::getUndefined<int32_t>()); case RES_SHORT: return AddShort(search::attribute::getUndefined<int16_t>()); + case RES_BOOL: return AddByte(0); case RES_BYTE: return AddByte(search::attribute::getUndefined<int8_t>()); case RES_FLOAT: return AddFloat(search::attribute::getUndefined<float>()); case RES_DOUBLE: return AddDouble(search::attribute::getUndefined<double>()); case RES_INT64: return AddInt64(search::attribute::getUndefined<int64_t>()); - case RES_STRING: return AddString(NULL, 0); - case RES_DATA: return AddData(NULL, 0); + case RES_STRING: return AddString(nullptr, 0); + case RES_DATA: return AddData(nullptr, 0); case RES_XMLSTRING: case RES_JSONSTRING: case RES_FEATUREDATA: - case RES_LONG_STRING: return AddLongString(NULL, 0); - case RES_TENSOR: return AddSerializedTensor(NULL, 0); - case RES_LONG_DATA: return AddLongData(NULL, 0); + case RES_LONG_STRING: return AddLongString(nullptr, 0); + case RES_TENSOR: return AddSerializedTensor(nullptr, 0); + case RES_LONG_DATA: return AddLongData(nullptr, 0); } } return AddInteger(0); // to provoke error condition @@ -271,7 +270,7 @@ ResultPacker::GetDocsumBlob(const char **buf, uint32_t *buflen) _resClass->GetNumEntries() - _entryIdx); } if (_error) { - *buf = NULL; + *buf = nullptr; *buflen = 0; return false; } else { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index 7cf1e65fbc0..9992d782d6e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -12,13 +12,11 @@ namespace search::docsummary { SummaryFeaturesDFW::SummaryFeaturesDFW() : - _env(NULL) + _env(nullptr) { } -SummaryFeaturesDFW::~SummaryFeaturesDFW() -{ -} +SummaryFeaturesDFW::~SummaryFeaturesDFW() = default; void SummaryFeaturesDFW::init(IDocsumEnvironment * env) @@ -30,8 +28,7 @@ static vespalib::string _G_cached("vespa.summaryFeatures.cached"); static vespalib::Memory _M_cached("vespa.summaryFeatures.cached"); void -SummaryFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) +SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) { if ( ! state->_summaryFeatures) { state->_callback.FillSummaryFeatures(state, _env); @@ -41,7 +38,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState } const FeatureSet::StringVector &names = state->_summaryFeatures->getNames(); const feature_t *values = state->_summaryFeatures->getFeaturesByDocId(docid); - if (type == RES_FEATUREDATA && values != NULL) { + if (type == RES_FEATUREDATA && values != nullptr) { vespalib::slime::Cursor& obj = target.insertObject(); for (uint32_t i = 0; i < names.size(); ++i) { vespalib::Memory name(names[i].c_str(), names[i].size()); @@ -55,7 +52,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState return; } vespalib::JSONStringer & json(state->_jsonStringer); - if (values != NULL) { + if (values != nullptr) { json.clear(); json.beginObject(); for (uint32_t i = 0; i < names.size(); ++i) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h index e417e89cf04..f1452d4c0a9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h @@ -10,7 +10,7 @@ namespace search::docsummary { class IDocsumEnvironment; -class FeaturesDFW : public IDocsumFieldWriter +class FeaturesDFW : public IDocsumFW { protected: void featureDump(vespalib::JSONStringer & json, vespalib::stringref name, double feature); @@ -29,7 +29,7 @@ public: ~SummaryFeaturesDFW() override; void init(IDocsumEnvironment * env); bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, + void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 290cc45648a..91a7fd45061 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -9,6 +9,7 @@ #include <vespa/document/annotation/spantreevisitor.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/boolfieldvalue.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/doublefieldvalue.h> @@ -42,6 +43,7 @@ using document::Annotation; using document::AnnotationReferenceFieldValue; using document::ArrayDataType; using document::ArrayFieldValue; +using document::BoolFieldValue; using document::ByteFieldValue; using document::DataType; using document::Document; @@ -229,7 +231,7 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor FieldValue::UP _field_value; FieldValueConverter &_structuredFieldConverter; - virtual void visit(const ArrayFieldValue &value) override { + void visit(const ArrayFieldValue &value) override { _field_value = _structuredFieldConverter.convert(value); } @@ -237,17 +239,18 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor void visitPrimitive(const T &t) { _field_value.reset(t.clone()); } - virtual void visit(const IntFieldValue &value) override { visitPrimitive(value); } - virtual void visit(const LongFieldValue &value) override { visitPrimitive(value); } - virtual void visit(const ShortFieldValue &value) override { visitPrimitive(value); } - virtual void visit(const ByteFieldValue &value) override { + void visit(const IntFieldValue &value) override { visitPrimitive(value); } + void visit(const LongFieldValue &value) override { visitPrimitive(value); } + void visit(const ShortFieldValue &value) override { visitPrimitive(value); } + void visit(const BoolFieldValue &value) override { visitPrimitive(value); } + void visit(const ByteFieldValue &value) override { int8_t signedValue = value.getAsByte(); _field_value.reset(new ShortFieldValue(signedValue)); } - virtual void visit(const DoubleFieldValue &value) override { visitPrimitive(value); } - virtual void visit(const FloatFieldValue &value) override { visitPrimitive(value); } + void visit(const DoubleFieldValue &value) override { visitPrimitive(value); } + void visit(const FloatFieldValue &value) override { visitPrimitive(value); } - virtual void visit(const StringFieldValue &value) override { + void visit(const StringFieldValue &value) override { if (_tokenize) { SummaryHandler handler(value.getValue(), _str); handleIndexingTerms(handler, value); @@ -256,33 +259,29 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor } } - virtual void visit(const AnnotationReferenceFieldValue & v ) override { + void visit(const AnnotationReferenceFieldValue & v ) override { _field_value = _structuredFieldConverter.convert(v); } - virtual void visit(const Document & v) override { + void visit(const Document & v) override { _field_value = _structuredFieldConverter.convert(v); } - virtual void - visit(const PredicateFieldValue &value) override - { + void visit(const PredicateFieldValue &value) override { _str << value.toString(); } - virtual void - visit(const RawFieldValue &value) override - { + void visit(const RawFieldValue &value) override { visitPrimitive(value); } - virtual void visit(const MapFieldValue & v) override { + void visit(const MapFieldValue & v) override { _field_value = _structuredFieldConverter.convert(v); } - virtual void visit(const StructFieldValue &value) override { + void visit(const StructFieldValue &value) override { if (*value.getDataType() == *SearchDataType::URI) { FieldValue::UP uriAllValue = value.getValue("all"); - if (uriAllValue.get() != NULL && + if (uriAllValue && uriAllValue->inherits(IDENTIFIABLE_CLASSID(StringFieldValue))) { uriAllValue->accept(*this); @@ -292,11 +291,11 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor _field_value = _structuredFieldConverter.convert(value); } - virtual void visit(const WeightedSetFieldValue &value) override { + void visit(const WeightedSetFieldValue &value) override { _field_value = _structuredFieldConverter.convert(value); } - virtual void visit(const TensorFieldValue &value) override { + void visit(const TensorFieldValue &value) override { visitPrimitive(value); } @@ -315,7 +314,7 @@ public: if (_field_value.get()) { return std::move(_field_value); } - return FieldValue::UP(new StringFieldValue(_str.str())); + return std::make_unique<StringFieldValue>(_str.str()); } }; @@ -323,7 +322,7 @@ SummaryFieldValueConverter::SummaryFieldValueConverter(bool tokenize, FieldValue : _str(), _tokenize(tokenize), _structuredFieldConverter(subConverter) {} -SummaryFieldValueConverter::~SummaryFieldValueConverter() {} +SummaryFieldValueConverter::~SummaryFieldValueConverter() = default; using namespace vespalib::slime::convenience; @@ -331,14 +330,14 @@ class SlimeFiller : public ConstFieldValueVisitor { Inserter &_inserter; bool _tokenize; - virtual void visit(const AnnotationReferenceFieldValue & v ) override { + void visit(const AnnotationReferenceFieldValue & v ) override { (void)v; Cursor &c = _inserter.insertObject(); Memory key("error"); Memory val("cannot convert from annotation reference field"); c.setString(key, val); } - virtual void visit(const Document & v) override { + void visit(const Document & v) override { (void)v; Cursor &c = _inserter.insertObject(); Memory key("error"); @@ -346,7 +345,7 @@ class SlimeFiller : public ConstFieldValueVisitor { c.setString(key, val); } - virtual void visit(const MapFieldValue & v) override { + void visit(const MapFieldValue & v) override { Cursor &a = _inserter.insertArray(); Symbol keysym = a.resolve("key"); Symbol valsym = a.resolve("value"); @@ -364,7 +363,7 @@ class SlimeFiller : public ConstFieldValueVisitor { } } - virtual void visit(const ArrayFieldValue &value) override { + void visit(const ArrayFieldValue &value) override { Cursor &a = _inserter.insertArray(); if (value.size() > 0) { ArrayInserter ai(a); @@ -375,7 +374,7 @@ class SlimeFiller : public ConstFieldValueVisitor { } } - virtual void visit(const StringFieldValue &value) override { + void visit(const StringFieldValue &value) override { if (_tokenize) { asciistream tmp; SummaryHandler handler(value.getValue(), tmp); @@ -386,48 +385,48 @@ class SlimeFiller : public ConstFieldValueVisitor { } } - virtual void visit(const IntFieldValue &value) override { + void visit(const IntFieldValue &value) override { int32_t v = value.getValue(); _inserter.insertLong(v); } - virtual void visit(const LongFieldValue &value) override { + void visit(const LongFieldValue &value) override { int64_t v = value.getValue(); _inserter.insertLong(v); } - virtual void visit(const ShortFieldValue &value) override { + void visit(const ShortFieldValue &value) override { int16_t v = value.getValue(); _inserter.insertLong(v); } - virtual void visit(const ByteFieldValue &value) override { + void visit(const ByteFieldValue &value) override { int8_t v = value.getAsByte(); _inserter.insertLong(v); } - virtual void visit(const DoubleFieldValue &value) override { + void visit(const BoolFieldValue &value) override { + bool v = value.getValue(); + _inserter.insertBool(v); + } + void visit(const DoubleFieldValue &value) override { double v = value.getValue(); _inserter.insertDouble(v); } - virtual void visit(const FloatFieldValue &value) override { + void visit(const FloatFieldValue &value) override { float v = value.getValue(); _inserter.insertDouble(v); } - virtual void - visit(const PredicateFieldValue &value) override - { + void visit(const PredicateFieldValue &value) override { vespalib::slime::inject(value.getSlime().get(), _inserter); } - virtual void - visit(const RawFieldValue &value) override - { + void visit(const RawFieldValue &value) override { std::pair<const char *, size_t> buf = value.getAsRaw(); _inserter.insertData(Memory(buf.first, buf.second)); } - virtual void visit(const StructFieldValue &value) override { + void visit(const StructFieldValue &value) override { if (*value.getDataType() == *SearchDataType::URI) { FieldValue::UP uriAllValue = value.getValue("all"); - if (uriAllValue.get() != NULL && + if (uriAllValue && uriAllValue->inherits(IDENTIFIABLE_CLASSID(StringFieldValue))) { uriAllValue->accept(*this); @@ -444,7 +443,7 @@ class SlimeFiller : public ConstFieldValueVisitor { } } - virtual void visit(const WeightedSetFieldValue &value) override { + void visit(const WeightedSetFieldValue &value) override { Cursor &a = _inserter.insertArray(); if (value.size() > 0) { Symbol isym = a.resolve("item"); @@ -460,7 +459,7 @@ class SlimeFiller : public ConstFieldValueVisitor { } } - virtual void visit(const TensorFieldValue &value) override { + void visit(const TensorFieldValue &value) override { const auto &tensor = value.getAsTensorPtr(); vespalib::nbostream s; if (tensor) { @@ -495,7 +494,7 @@ public: search::RawBuf rbuf(4096); search::SlimeOutputRawBufAdapter adapter(rbuf); vespalib::slime::BinaryFormat::encode(slime, adapter); - return FieldValue::UP(new RawFieldValue(rbuf.GetDrainPos(), rbuf.GetUsedLen())); + return std::make_unique<RawFieldValue>(rbuf.GetDrainPos(), rbuf.GetUsedLen()); } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp index 121520c4d03..a0efda07f04 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp @@ -27,16 +27,13 @@ TextExtractorDFW::init(const vespalib::string & fieldName, const vespalib::strin } void -TextExtractorDFW::insertField(uint32_t, - GeneralResult *gres, - GetDocsumsState *state, - ResType, +TextExtractorDFW::insertField(uint32_t, GeneralResult *gres, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) { vespalib::string extracted; ResEntry * entry = gres->GetEntryFromEnumValue(_inputFieldEnum); - if (entry != NULL) { - const char * buf = NULL; + if (entry != nullptr) { + const char * buf = nullptr; uint32_t buflen = 0; entry->_resolve_field(&buf, &buflen, &state->_docSumFieldSpace); // extract the text diff --git a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp index 9cd5c58f971..074cc1cadf1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp @@ -10,45 +10,6 @@ LOG_SETUP(".searchlib.docsummary.urlresult"); namespace search::docsummary { -urlresult::urlresult(uint32_t partition, uint32_t docid, HitRank metric) - : _partition(partition), - _docid(docid), - _metric(metric) -{ } - - -urlresult::~urlresult() { } - - -/*===============================================================*/ - - -badurlresult::badurlresult() - : urlresult(0, 0, 0) -{ } - - -badurlresult::badurlresult(uint32_t partition, uint32_t docid, HitRank metric) - : urlresult(partition, docid, metric) -{ } - - -badurlresult::~badurlresult() { } - - -int -badurlresult::unpack(const char *buf, const size_t buflen) -{ - (void) buf; - (void) buflen; - LOG(warning, "badurlresult::unpack"); - return 0; -} - - -/*===============================================================*/ - - void GeneralResult::AllocEntries(uint32_t buflen, bool inplace) { @@ -60,10 +21,10 @@ GeneralResult::AllocEntries(uint32_t buflen, bool inplace) if (cnt > 0) { _entrycnt = cnt; _entries = (ResEntry *) malloc(needMem); - assert(_entries != NULL); + assert(_entries != nullptr); if (inplace) { - _buf = NULL; - _bufEnd = NULL; + _buf = nullptr; + _bufEnd = nullptr; } else { _buf = ((char *)_entries) + cnt * sizeof(ResEntry); _bufEnd = _buf + buflen + 1; @@ -71,20 +32,19 @@ GeneralResult::AllocEntries(uint32_t buflen, bool inplace) memset(_entries, 0, cnt * sizeof(ResEntry)); } else { _entrycnt = 0; - _entries = NULL; - _buf = NULL; - _bufEnd = NULL; + _entries = nullptr; + _buf = nullptr; + _bufEnd = nullptr; } } - void GeneralResult::FreeEntries() { uint32_t cnt = _entrycnt; - // (_buf == NULL) <=> (_inplace_unpack() || (cnt == 0)) - if (_buf != NULL) { + // (_buf == nullptr) <=> (_inplace_unpack() || (cnt == 0)) + if (_buf != nullptr) { for (uint32_t i = 0; i < cnt; i++) { if (ResultConfig::IsVariableSize(_entries[i]._type) && !InBuf(_entries[i]._stringval)) @@ -94,41 +54,32 @@ GeneralResult::FreeEntries() free(_entries); // free '_entries'/'_buf' chunk } - - -GeneralResult::GeneralResult(const ResultClass *resClass, - uint32_t partition, uint32_t docid, - HitRank metric) - : urlresult(partition, docid, metric), - _resClass(resClass), +GeneralResult::GeneralResult(const ResultClass *resClass) + : _resClass(resClass), _entrycnt(0), - _entries(NULL), - _buf(NULL), - _bufEnd(NULL) + _entries(nullptr), + _buf(nullptr), + _bufEnd(nullptr) { } - GeneralResult::~GeneralResult() { FreeEntries(); } - ResEntry * GeneralResult::GetEntry(uint32_t idx) { - return (idx < _entrycnt) ? &_entries[idx] : NULL; + return (idx < _entrycnt) ? &_entries[idx] : nullptr; } - ResEntry * GeneralResult::GetEntry(const char *name) { int idx = _resClass->GetIndexFromName(name); - return (idx >= 0 && (uint32_t)idx < _entrycnt) ? - &_entries[idx] : NULL; + return (idx >= 0 && (uint32_t)idx < _entrycnt) ? &_entries[idx] : nullptr; } @@ -136,398 +87,17 @@ ResEntry * GeneralResult::GetEntryFromEnumValue(uint32_t value) { int idx = _resClass->GetIndexFromEnumValue(value); - - return (idx >= 0 && (uint32_t)idx < _entrycnt) ? - &_entries[idx] : NULL; + return (idx >= 0 && (uint32_t)idx < _entrycnt) ? &_entries[idx] : nullptr; } - -int -GeneralResult::unpack(const char *buf, const size_t buflen) -{ - bool rc = true; - const char *ebuf = buf + buflen; // Ref to first after buffer - const char *p = buf; // current position in buffer - - if (_entries != NULL) - FreeEntries(); - - AllocEntries(buflen); - - for (uint32_t i = 0; rc && i < _entrycnt; i++) { - const ResConfigEntry *entry = _resClass->GetEntry(i); - - switch (entry->_type) { - - case RES_INT: { - - if (p + sizeof(_entries[i]._intval) <= ebuf) { - - memcpy(&_entries[i]._intval, p, sizeof(_entries[i]._intval)); - _entries[i]._type = RES_INT; - p += sizeof(_entries[i]._intval); - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(..._intval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_SHORT: { - - uint16_t shortval; - if (p + sizeof(shortval) <= ebuf) { - - memcpy(&shortval, p, sizeof(shortval)); - _entries[i]._intval = (uint32_t)shortval; - _entries[i]._type = RES_INT; // type promotion - p += sizeof(shortval); - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(shortval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_BYTE: { - - uint8_t byteval; - if (p + sizeof(byteval) <= ebuf) { - - memcpy(&byteval, p, sizeof(byteval)); - _entries[i]._intval = (uint32_t)byteval; - _entries[i]._type = RES_INT; // type promotion - p += sizeof(byteval); - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(byteval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_FLOAT: { - - float floatval; - if (p + sizeof(floatval) <= ebuf) { - - memcpy(&floatval, p, sizeof(floatval)); - _entries[i]._doubleval = (double)floatval; - _entries[i]._type = RES_DOUBLE; // type promotion - p += sizeof(floatval); - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(floatval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_DOUBLE: { - - if (p + sizeof(_entries[i]._doubleval) <= ebuf) { - - memcpy(&_entries[i]._doubleval, p, sizeof(_entries[i]._doubleval)); - _entries[i]._type = RES_DOUBLE; - p += sizeof(_entries[i]._doubleval); - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(..._doubleval) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_INT64: { - - if (p + sizeof(_entries[i]._int64val) <= ebuf) { - - memcpy(&_entries[i]._int64val, p, sizeof(_entries[i]._int64val)); - _entries[i]._type = RES_INT64; - p += sizeof(_entries[i]._int64val); - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(..._int64val) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_STRING: { - - uint16_t slen; - if (p + sizeof(slen) <= ebuf) { - - memcpy(&slen, p, sizeof(slen)); - p += sizeof(slen); - - if (p + slen <= ebuf) { - - _entries[i]._stringval = _buf + (p - buf); - memcpy(_entries[i]._stringval, p, slen); - _entries[i]._stringval[slen] = '\0'; - _entries[i]._stringlen = slen; - _entries[i]._type = RES_STRING; - p += slen; - - } else { - - LOG(debug, "GeneralResult::unpack: p + slen > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(slen) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_DATA: { - - uint16_t dlen; - if (p + sizeof(dlen) <= ebuf) { - - memcpy(&dlen, p, sizeof(dlen)); - p += sizeof(dlen); - - if (p + dlen <= ebuf) { - - _entries[i]._dataval = _buf + (p - buf); - memcpy(_entries[i]._dataval, p, dlen); - _entries[i]._dataval[dlen] = '\0'; // just in case. - _entries[i]._datalen = dlen; - _entries[i]._type = RES_DATA; - p += dlen; - - } else { - - LOG(debug, "GeneralResult::unpack: p + dlen > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(dlen) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_XMLSTRING: - case RES_JSONSTRING: - case RES_FEATUREDATA: - case RES_LONG_STRING: { - - uint32_t lslen; - bool compressed; - if (p + sizeof(lslen) <= ebuf) { - - memcpy(&lslen, p, sizeof(lslen)); - p += sizeof(lslen); - - compressed = ((lslen & 0x80000000) != 0); - lslen &= 0x7fffffff; - - if (p + lslen <= ebuf) { - - if (compressed) { // COMPRESSED - uint32_t realLen = 0; - if (lslen >= sizeof(realLen)) - memcpy(&realLen, p, sizeof(realLen)); - else - LOG(warning, "Cannot uncompress docsum field %s; docsum field meta-data incomplete", - entry->_bindname.c_str()); - if (realLen > 0) { - _entries[i]._stringval = new char[realLen + 1]; - } - if (_entries[i]._stringval != NULL) { - uLongf rlen = realLen; - if ((uncompress((Bytef *)_entries[i]._stringval, &rlen, - (const Bytef *)(p + sizeof(realLen)), - lslen - sizeof(realLen)) == Z_OK) && - rlen == realLen) { - assert(rlen == realLen); - - // COMPRESSED LONG STRING FIELD OK - _entries[i]._stringval[realLen] = '\0'; - _entries[i]._stringlen = realLen; - - } else { - LOG(warning, "Cannot uncompress docsum field %s; decompression error", - entry->_bindname.c_str()); - delete [] _entries[i]._stringval; - _entries[i]._stringval = NULL; - } - } - // insert empty field if decompress failed - if (_entries[i]._stringval == NULL) { - _entries[i]._stringval = _buf + (p - buf); - _entries[i]._stringval[0] = '\0'; - _entries[i]._stringlen = 0; - } - - } else { // UNCOMPRESSED - - _entries[i]._stringval = _buf + (p - buf); - memcpy(_entries[i]._stringval, p, lslen); - _entries[i]._stringval[lslen] = '\0'; - _entries[i]._stringlen = lslen; - - } - _entries[i]._type = RES_STRING; // type normalization - p += lslen; - - } else { - - LOG(debug, "GeneralResult::unpack: p + lslen > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(lslen) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - case RES_TENSOR: - case RES_LONG_DATA: { - - uint32_t ldlen; - bool compressed; - if (p + sizeof(ldlen) <= ebuf) { - - memcpy(&ldlen, p, sizeof(ldlen)); - p += sizeof(ldlen); - - compressed = ((ldlen & 0x80000000) != 0); - ldlen &= 0x7fffffff; - - if (p + ldlen <= ebuf) { - - if (compressed) { // COMPRESSED - uint32_t realLen = 0; - if (ldlen >= sizeof(realLen)) - memcpy(&realLen, p, sizeof(realLen)); - else - LOG(warning, "Cannot uncompress docsum field %s; docsum field meta-data incomplete", - entry->_bindname.c_str()); - if (realLen > 0) { - _entries[i]._dataval = new char [realLen + 1]; - } - if (_entries[i]._dataval != NULL) { - uLongf rlen = realLen; - if ((uncompress((Bytef *)_entries[i]._dataval, &rlen, - (const Bytef *)(p + sizeof(realLen)), - ldlen - sizeof(realLen)) == Z_OK) && - rlen == realLen) { - assert(rlen == realLen); - - // COMPRESSED LONG DATA FIELD OK - _entries[i]._dataval[realLen] = '\0'; - _entries[i]._datalen = realLen; - - } else { - LOG(warning, "Cannot uncompress docsum field %s; decompression error", - entry->_bindname.c_str()); - delete [] _entries[i]._dataval; - _entries[i]._dataval = NULL; - } - } - - // insert empty field if decompress failed - if (_entries[i]._dataval == NULL) { - _entries[i]._dataval = _buf + (p - buf); - _entries[i]._dataval[0] = '\0'; - _entries[i]._datalen = 0; - } - - } else { // UNCOMPRESSED - - _entries[i]._dataval = _buf + (p - buf); - memcpy(_entries[i]._dataval, p, ldlen); - _entries[i]._dataval[ldlen] = '\0'; // just in case - _entries[i]._datalen = ldlen; - - } - _entries[i]._type = RES_DATA; // type normalization - p += ldlen; - - } else { - - LOG(debug, "GeneralResult::unpack: p + ldlen > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - - } else { - - LOG(debug, "GeneralResult::unpack: p + sizeof(ldlen) > ebuf"); - LOG(error, "Document summary too short, couldn't unpack"); - rc = false; - } - break; - } - - default: - LOG(warning, "GeneralResult::unpack: no such type:%d", entry->_type); - LOG(error, "Incorrect type in document summary, couldn't unpack"); - rc = false; - break; - } // END -- switch (entry->_type) { - } // END -- for (uint32_t i = 0; rc && i < _entrycnt; i++) { - - if (rc && p != ebuf) { - LOG(debug, "GeneralResult::unpack: p:%p != ebuf:%p", p, ebuf); - LOG(error, "Document summary too long, couldn't unpack."); - rc = false; - } - - if (rc) - return 0; // SUCCESS - - // clean up on failure - FreeEntries(); - _entrycnt = 0; - _entries = NULL; - _buf = NULL; - _bufEnd = NULL; - - return -1; // FAIL -} - - bool -GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) +GeneralResult::unpack(const char *buf, const size_t buflen) { bool rc = true; const char *ebuf = buf + buflen; // Ref to first after buffer const char *p = buf; // current position in buffer - if (_entries != NULL) + if (_entries != nullptr) FreeEntries(); AllocEntries(buflen, true); @@ -538,17 +108,12 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) switch (entry->_type) { case RES_INT: { - if (p + sizeof(_entries[i]._intval) <= ebuf) { - memcpy(&_entries[i]._intval, p, sizeof(_entries[i]._intval)); _entries[i]._type = RES_INT; p += sizeof(_entries[i]._intval); - } else { - - LOG(debug, - "GeneralResult::_inplace_unpack: p + sizeof(..._intval) > ebuf"); + LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(..._intval) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; } @@ -556,39 +121,29 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } case RES_SHORT: { - uint16_t shortval; if (p + sizeof(shortval) <= ebuf) { - memcpy(&shortval, p, sizeof(shortval)); _entries[i]._intval = (uint32_t)shortval; _entries[i]._type = RES_INT; // type promotion p += sizeof(shortval); - } else { - - LOG(debug, - "GeneralResult::_inplace_unpack: p + sizeof(shortval) > ebuf"); + LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(shortval) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; } break; } - + case RES_BOOL: case RES_BYTE: { - uint8_t byteval; if (p + sizeof(byteval) <= ebuf) { - memcpy(&byteval, p, sizeof(byteval)); _entries[i]._intval = (uint32_t)byteval; _entries[i]._type = RES_INT; // type promotion p += sizeof(byteval); - } else { - - LOG(debug, - "GeneralResult::_inplace_unpack: p + sizeof(byteval) > ebuf"); + LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(byteval) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; } @@ -596,17 +151,13 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } case RES_FLOAT: { - float floatval; if (p + sizeof(floatval) <= ebuf) { - memcpy(&floatval, p, sizeof(floatval)); _entries[i]._doubleval = (double)floatval; _entries[i]._type = RES_DOUBLE; // type promotion p += sizeof(floatval); - } else { - LOG(debug, "GeneralResult::unpack: p + sizeof(floatval) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; @@ -615,15 +166,11 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } case RES_DOUBLE: { - if (p + sizeof(_entries[i]._doubleval) <= ebuf) { - memcpy(&_entries[i]._doubleval, p, sizeof(_entries[i]._doubleval)); _entries[i]._type = RES_DOUBLE; p += sizeof(_entries[i]._doubleval); - } else { - LOG(debug, "GeneralResult::unpack: p + sizeof(..._doubleval) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; @@ -632,15 +179,11 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } case RES_INT64: { - if (p + sizeof(_entries[i]._int64val) <= ebuf) { - memcpy(&_entries[i]._int64val, p, sizeof(_entries[i]._int64val)); _entries[i]._type = RES_INT64; p += sizeof(_entries[i]._int64val); - } else { - LOG(debug, "GeneralResult::unpack: p + sizeof(..._int64val) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; @@ -649,29 +192,21 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } case RES_STRING: { - uint16_t slen; if (p + sizeof(slen) <= ebuf) { - memcpy(&slen, p, sizeof(slen)); p += sizeof(slen); - if (p + slen <= ebuf) { - _entries[i]._stringval = const_cast<char *>(p); _entries[i]._stringlen = slen; _entries[i]._type = RES_STRING; p += slen; - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + slen > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; } - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(slen) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; @@ -680,29 +215,21 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } case RES_DATA: { - uint16_t dlen; if (p + sizeof(dlen) <= ebuf) { - memcpy(&dlen, p, sizeof(dlen)); p += sizeof(dlen); - if (p + dlen <= ebuf) { - _entries[i]._dataval = const_cast<char *>(p); _entries[i]._datalen = dlen; _entries[i]._type = RES_DATA; p += dlen; - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + dlen > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; } - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(dlen) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; @@ -714,32 +241,23 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) case RES_JSONSTRING: case RES_FEATUREDATA: case RES_LONG_STRING: { - uint32_t flen; uint32_t lslen; if (p + sizeof(flen) <= ebuf) { - memcpy(&flen, p, sizeof(flen)); p += sizeof(flen); - lslen = flen & 0x7fffffff; - if (p + lslen <= ebuf) { - _entries[i]._stringval = const_cast<char *>(p); _entries[i]._stringlen = flen; // with compression flag _entries[i]._type = RES_STRING; // type normalization p += lslen; - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + lslen > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; } - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(lslen) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; @@ -748,32 +266,23 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } case RES_TENSOR : case RES_LONG_DATA: { - uint32_t flen; uint32_t ldlen; if (p + sizeof(flen) <= ebuf) { - memcpy(&flen, p, sizeof(flen)); p += sizeof(flen); - ldlen = flen & 0x7fffffff; - if (p + ldlen <= ebuf) { - _entries[i]._dataval = const_cast<char *>(p); _entries[i]._datalen = flen; // with compression flag _entries[i]._type = RES_DATA; // type normalization p += ldlen; - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + ldlen > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; } - } else { - LOG(debug, "GeneralResult::_inplace_unpack: p + sizeof(ldlen) > ebuf"); LOG(error, "Document summary too short, couldn't unpack"); rc = false; @@ -782,9 +291,7 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) } default: - LOG(warning, - "GeneralResult::_inplace_unpack: no such type:%d", - entry->_type); + LOG(warning, "GeneralResult::_inplace_unpack: no such type:%d", entry->_type); LOG(error, "Incorrect type in document summary, couldn't unpack"); rc = false; break; @@ -803,9 +310,9 @@ GeneralResult::_inplace_unpack(const char *buf, const size_t buflen) // clean up on failure FreeEntries(); _entrycnt = 0; - _entries = NULL; - _buf = NULL; - _bufEnd = NULL; + _entries = nullptr; + _buf = nullptr; + _bufEnd = nullptr; return false; // FAIL } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.h b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.h index 4d1fca0992d..a4cdd1b7f69 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/urlresult.h @@ -7,37 +7,7 @@ namespace search::docsummary { -class urlresult -{ -protected: - uint32_t _partition; - uint32_t _docid; - HitRank _metric; - -public: - urlresult(uint32_t partition, uint32_t docid, HitRank metric); - virtual ~urlresult(); - - virtual bool IsGeneral() const { return false; } - uint32_t GetPartition() const { return _partition; } - uint32_t GetDocID() const { return _docid; } - HitRank GetMetric() const { return _metric; } - virtual int unpack(const char *buf, const size_t buflen) = 0; -}; - - -class badurlresult : public urlresult -{ -public: - badurlresult(); - badurlresult(uint32_t partition, uint32_t docid, HitRank metric); - ~badurlresult() override; - - int unpack(const char *buf, const size_t buflen) override; -}; - - -class GeneralResult : public urlresult +class GeneralResult { private: GeneralResult(const GeneralResult &); @@ -49,31 +19,27 @@ private: char *_buf; // allocated in same chunk as _entries char *_bufEnd; // first byte after _buf - bool InBuf(void *pt) { - return ((char *)pt >= _buf && - (char *)pt < _bufEnd); + bool InBuf(const void *pt) const { + return ((const char *)pt >= _buf && + (const char *)pt < _bufEnd); } void AllocEntries(uint32_t buflen, bool inplace = false); void FreeEntries(); - bool _inplace_unpack(const char *buf, const size_t buflen); - public: - GeneralResult(const ResultClass *resClass, uint32_t partition, - uint32_t docid, HitRank metric); + GeneralResult(const ResultClass *resClass); ~GeneralResult(); const ResultClass *GetClass() const { return _resClass; } ResEntry *GetEntry(uint32_t idx); ResEntry *GetEntry(const char *name); ResEntry *GetEntryFromEnumValue(uint32_t val); - bool IsGeneral() const override { return true; } - int unpack(const char *buf, const size_t buflen) override; + bool unpack(const char *buf, const size_t buflen); bool inplaceUnpack(const DocsumStoreValue &value) { if (value.valid()) { - return _inplace_unpack(value.fieldsPt(), value.fieldsSz()); + return unpack(value.fieldsPt(), value.fieldsSz()); } else { return false; } diff --git a/staging_vespalib/src/tests/state_server/state_server_test.cpp b/staging_vespalib/src/tests/state_server/state_server_test.cpp index b688887c3fb..d4f665029cc 100644 --- a/staging_vespalib/src/tests/state_server/state_server_test.cpp +++ b/staging_vespalib/src/tests/state_server/state_server_test.cpp @@ -40,7 +40,7 @@ vespalib::string run_cmd(const vespalib::string &cmd) { } vespalib::string getPage(int port, const vespalib::string &path, const vespalib::string &extra_params = "") { - return run_cmd(make_string("curl -s %s http://localhost:%d%s", extra_params.c_str(), port, path.c_str())); + return run_cmd(make_string("curl -s %s 'http://localhost:%d%s'", extra_params.c_str(), port, path.c_str())); } vespalib::string getFull(int port, const vespalib::string &path) { return getPage(port, path, "-D -"); } @@ -123,6 +123,50 @@ TEST_FF("require that host is passed correctly", EchoHost(), HttpServer(0)) { EXPECT_EQUAL(default_result, run_cmd(make_string("curl -s http://localhost:%d/my/path -H \"Host:\"", f2.port()))); } +struct SamplingHandler : JsonGetHandler { + mutable std::mutex my_lock; + mutable vespalib::string my_host; + mutable vespalib::string my_path; + mutable std::map<vespalib::string,vespalib::string> my_params; + vespalib::string get(const vespalib::string &host, const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> ¶ms) const override + { + { + auto guard = std::lock_guard(my_lock); + my_host = host; + my_path = path; + my_params = params; + } + return "[]"; + } +}; + +TEST_FF("require that request parameters can be inspected", SamplingHandler(), HttpServer(0)) +{ + auto token = f2.repo().bind("/foo", f1); + EXPECT_EQUAL("[]", getPage(f2.port(), "/foo?a=b&x=y&z")); + { + auto guard = std::lock_guard(f1.my_lock); + EXPECT_EQUAL(f1.my_path, "/foo"); + EXPECT_EQUAL(f1.my_params.size(), 3u); + EXPECT_EQUAL(f1.my_params["a"], "b"); + EXPECT_EQUAL(f1.my_params["x"], "y"); + EXPECT_EQUAL(f1.my_params["z"], ""); + EXPECT_EQUAL(f1.my_params.size(), 3u); // "z" was present + } +} + +TEST_FF("require that request path is dequoted", SamplingHandler(), HttpServer(0)) +{ + auto token = f2.repo().bind("/[foo]", f1); + EXPECT_EQUAL("[]", getPage(f2.port(), "/%5bfoo%5D")); + { + auto guard = std::lock_guard(f1.my_lock); + EXPECT_EQUAL(f1.my_path, "/[foo]"); + EXPECT_EQUAL(f1.my_params.size(), 0u); + } +} + //----------------------------------------------------------------------------- TEST_FFFF("require that the state server wires the appropriate url prefixes", diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp index 99a66fccde5..2a3bb5b3e0e 100644 --- a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp +++ b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp @@ -8,7 +8,7 @@ namespace vespalib { void HttpServer::get(Portal::GetRequest req) { - vespalib::string json_result = _handler_repo.get(req.get_host(), req.get_uri(), {}); + vespalib::string json_result = _handler_repo.get(req.get_host(), req.get_path(), req.export_params()); if (json_result.empty()) { req.respond_with_error(404, "Not Found"); } else { diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp index 9272f7b0f94..35110be87ca 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp @@ -4,8 +4,7 @@ #include "xmlstream.h" #include <sstream> -namespace vespalib { -namespace xml { +namespace vespalib::xml { std::string XmlSerializable::toXml(const std::string& indent) const @@ -16,5 +15,4 @@ XmlSerializable::toXml(const std::string& indent) const return ost.str(); } -} // xml -} // vespalib +} diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h index c0d41910479..4c00a734b2b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h +++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h @@ -4,8 +4,7 @@ #include <string> -namespace vespalib { -namespace xml { +namespace vespalib::xml { class XmlOutputStream; @@ -26,8 +25,9 @@ public: virtual std::string toXml(const std::string& indent = "") const; }; -} // xml +} +namespace vespalib { // The XmlSerializable and XmlOutputStream is often used in header files // and is thus available in the vespalib namespace. To not pollute the // vespalib namespace with all the other classes, use diff --git a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java index 1d20a06d0da..bd149e5a41e 100644 --- a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java +++ b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java @@ -15,6 +15,8 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.Collections; import java.util.List; @@ -38,6 +40,7 @@ public class ApplicationMojo extends AbstractMojo { File applicationPackage = new File(project.getBasedir(), sourceDir); File applicationDestination = new File(project.getBasedir(), destinationDir); copyApplicationPackage(applicationPackage, applicationDestination); + addBuildMetaData(applicationDestination); File componentsDir = createComponentsDir(applicationDestination); copyModuleBundles(project.getBasedir(), componentsDir); @@ -50,6 +53,24 @@ public class ApplicationMojo extends AbstractMojo { } } + /** Writes meta data about this package if the destination directory exists, and the "vespaversion" property is set. */ + private void addBuildMetaData(File applicationDestination) throws MojoExecutionException { + String compileVersion = project.getProperties().getProperty("vespaversion"); + if ( ! applicationDestination.exists() || compileVersion == null) + return; + + String metaData = String.format("{\"compileVersion\": \"%s\",\n \"buildTime\": %d}", + compileVersion, + System.currentTimeMillis()); + try { + Files.write(applicationDestination.toPath().resolve("build-meta.json"), + metaData.getBytes(StandardCharsets.UTF_8)); + } + catch (IOException e) { + throw new MojoExecutionException("Failed writing compile version and build time.", e); + } + } + private void copyBundlesForSubModules(File componentsDir) throws MojoExecutionException { List<String> modules = emptyListIfNull(project.getModules()); for (String module : modules) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index 98d9061be02..cd35a204b00 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.StreamSupport; +import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; /** @@ -117,6 +118,9 @@ public class SiaUtils { public static List<AthenzService> findSiaServices(Path root) { String keyFileSuffix = ".key.pem"; Path keysDirectory = root.resolve("keys"); + if ( ! Files.exists(keysDirectory)) + return emptyList(); + try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(keysDirectory)) { return StreamSupport.stream(directoryStream.spliterator(), false) .map(path -> path.getFileName().toString()) diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java index 9337bb94c23..f69e937f294 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java @@ -11,8 +11,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import static java.util.Collections.emptyList; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; /** @@ -26,6 +28,7 @@ public class SiaUtilsTest { @Test public void it_finds_all_identity_names_from_files_in_sia_keys_directory() throws IOException { Path siaRoot = tempDirectory.getRoot().toPath(); + assertThat(SiaUtils.findSiaServices(siaRoot), is(emptyList())); Files.createDirectory(siaRoot.resolve("keys")); AthenzService fooService = new AthenzService("my.domain.foo"); Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, fooService)); diff --git a/vespalib/src/tests/portal/http_request/http_request_test.cpp b/vespalib/src/tests/portal/http_request/http_request_test.cpp index 6e1527efa4b..047fde5750c 100644 --- a/vespalib/src/tests/portal/http_request/http_request_test.cpp +++ b/vespalib/src/tests/portal/http_request/http_request_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/portal/http_request.h> +#include <vespa/vespalib/util/stringfmt.h> using namespace vespalib; using namespace vespalib::portal; @@ -117,4 +118,58 @@ TEST("require that header line must contain separator") { "missing separator\r\n")); } +TEST("require that uri parameters can be parsed") { + auto req = make_request("GET /my/path?foo=bar&baz HTTP/1.1\r\n\r\n"); + EXPECT_EQUAL(req.get_uri(), "/my/path?foo=bar&baz"); + EXPECT_EQUAL(req.get_path(), "/my/path"); + EXPECT_TRUE(req.has_param("foo")); + EXPECT_TRUE(!req.has_param("bar")); + EXPECT_TRUE(req.has_param("baz")); + EXPECT_EQUAL(req.get_param("foo"), "bar"); + EXPECT_EQUAL(req.get_param("bar"), ""); + EXPECT_EQUAL(req.get_param("baz"), ""); +} + +TEST("require that byte values in uri segments (path, key, value) are dequoted as expected") { + vespalib::string str = "0123456789aBcDeF"; + for (size_t a = 0; a < 16; ++a) { + for (size_t b = 0; b < 16; ++b) { + vespalib::string expect = " foo "; + expect.push_back((a * 16) + b); + expect.push_back((a * 16) + b); + expect.append(" bar "); + vespalib::string input = vespalib::make_string("+foo+%%%c%c%%%c%c+bar+", + str[a], str[b], str[a], str[b]); + vespalib::string uri = vespalib::make_string("%s?%s=%s&extra=yes", + input.c_str(), input.c_str(), input.c_str()); + auto req = make_request(vespalib::make_string("GET %s HTTP/1.1\r\n\r\n", + uri.c_str())); + EXPECT_EQUAL(req.get_uri(), uri); + EXPECT_EQUAL(req.get_path(), expect); + EXPECT_TRUE(req.has_param(expect)); + EXPECT_EQUAL(req.get_param(expect), expect); + EXPECT_TRUE(req.has_param("extra")); + EXPECT_EQUAL(req.get_param("extra"), "yes"); + } + } +} + +TEST("require that percent character becomes plain if not followed by exactly 2 hex digits") { + auto req = make_request("GET %/5%5:%@5%5G%`5%5g%5?% HTTP/1.1\r\n\r\n"); + EXPECT_EQUAL(req.get_path(), "%/5%5:%@5%5G%`5%5g%5"); + EXPECT_TRUE(req.has_param("%")); +} + +TEST("require that last character of uri segments (path, key, value) can be quoted") { + auto req = make_request("GET /%41?%42=%43 HTTP/1.1\r\n\r\n"); + EXPECT_EQUAL(req.get_path(), "/A"); + EXPECT_EQUAL(req.get_param("B"), "C"); +} + +TEST("require that additional query and key/value separators are not special") { + auto req = make_request("GET /?" "?== HTTP/1.1\r\n\r\n"); + EXPECT_EQUAL(req.get_path(), "/"); + EXPECT_EQUAL(req.get_param("?"), "="); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/portal/portal_test.cpp b/vespalib/src/tests/portal/portal_test.cpp index 299340fd131..ee5d10a313a 100644 --- a/vespalib/src/tests/portal/portal_test.cpp +++ b/vespalib/src/tests/portal/portal_test.cpp @@ -346,4 +346,40 @@ TEST_MT_FFF("require that portal destruction waits for request completion", 3, //----------------------------------------------------------------------------- +TEST("require that query parameters can be inspected") { + auto portal = Portal::create(null_crypto(), 0); + MyGetHandler handler([](Portal::GetRequest request) + { + EXPECT_EQUAL(request.get_uri(), "/test?a=b&x=y"); + EXPECT_EQUAL(request.get_path(), "/test"); + EXPECT_TRUE(request.has_param("a")); + EXPECT_TRUE(request.has_param("x")); + EXPECT_TRUE(!request.has_param("b")); + EXPECT_EQUAL(request.get_param("a"), "b"); + EXPECT_EQUAL(request.get_param("x"), "y"); + EXPECT_EQUAL(request.get_param("b"), ""); + auto params = request.export_params(); + EXPECT_EQUAL(params.size(), 2u); + EXPECT_EQUAL(params["a"], "b"); + EXPECT_EQUAL(params["x"], "y"); + request.respond_with_content("a", "b"); + }); + auto bound = portal->bind("/test", handler); + auto result = fetch(portal->listen_port(), null_crypto(), "/test?a=b&x=y"); + EXPECT_EQUAL(result, make_expected_response("a", "b")); +} + +TEST("require that request path is dequoted before handler dispatching") { + auto portal = Portal::create(null_crypto(), 0); + MyGetHandler handler([](Portal::GetRequest request) + { + EXPECT_EQUAL(request.get_uri(), "/%5btest%5D"); + EXPECT_EQUAL(request.get_path(), "/[test]"); + request.respond_with_content("a", "b"); + }); + auto bound = portal->bind("/[test]", handler); + auto result = fetch(portal->listen_port(), null_crypto(), "/%5btest%5D"); + EXPECT_EQUAL(result, make_expected_response("a", "b")); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/portal/http_request.cpp b/vespalib/src/vespa/vespalib/portal/http_request.cpp index d49fc2e70f4..abd690897c6 100644 --- a/vespalib/src/vespa/vespalib/portal/http_request.cpp +++ b/vespalib/src/vespa/vespalib/portal/http_request.cpp @@ -15,11 +15,11 @@ void strip_cr(vespalib::string &str) { } } -std::vector<vespalib::string> split(vespalib::stringref str, vespalib::stringref sep) { +std::vector<vespalib::string> split(vespalib::stringref str, char sep) { vespalib::string token; std::vector<vespalib::string> list; for (char c: str) { - if (sep.find(c) == vespalib::stringref::npos) { + if (c != sep) { token.push_back(c); } else if (!token.empty()) { list.push_back(token); @@ -32,6 +32,49 @@ std::vector<vespalib::string> split(vespalib::stringref str, vespalib::stringref return list; } +int decode_hex_digit(char c) { + if ((c >= '0') && (c <= '9')) { + return (c - '0'); + } + if ((c >= 'a') && (c <= 'f')) { + return ((c - 'a') + 10); + } + if ((c >= 'A') && (c <= 'F')) { + return ((c - 'A') + 10); + } + return -1; +} + +int decode_hex_num(vespalib::stringref src, size_t idx) { + if (src.size() < (idx + 2)) { + return -1; + } + int a = decode_hex_digit(src[idx]); + int b = decode_hex_digit(src[idx + 1]); + if ((a < 0) || (b < 0)) { + return -1; + } + return ((a << 4) | b); +} + +vespalib::string dequote(vespalib::stringref src) { + vespalib::string dst; + for (size_t idx = 0; idx < src.size(); ++idx) { + char c = src[idx]; + if (c == '+') { + c = ' '; + } else if (c == '%') { + int x = decode_hex_num(src, idx + 1); + if (x >= 0) { + c = x; + idx += 2; + } + } + dst.push_back(c); + } + return dst; +} + } // namespace vespalib::portal::<unnamed> void @@ -49,13 +92,30 @@ HttpRequest::set_error() void HttpRequest::handle_request_line(const vespalib::string &line) { - auto parts = split(line, " "); + auto parts = split(line, ' '); if (parts.size() != 3) { return set_error(); // malformed request line } _method = parts[0]; _uri = parts[1]; _version = parts[2]; + size_t query_sep = _uri.find("?"); + if (query_sep == vespalib::string::npos) { + _path = dequote(_uri); + } else { + _path = dequote(_uri.substr(0, query_sep)); + auto query = split(_uri.substr(query_sep + 1), '&'); + for (const auto ¶m: query) { + size_t value_sep = param.find("="); + if (value_sep == vespalib::string::npos) { + _params[dequote(param)] = ""; + } else { + auto key = param.substr(0, value_sep); + auto value = param.substr(value_sep + 1); + _params[dequote(key)] = dequote(value); + } + } + } } void @@ -163,4 +223,20 @@ HttpRequest::get_header(const vespalib::string &name) const return pos->second; } +bool +HttpRequest::has_param(const vespalib::string &name) const +{ + return (_params.find(name) != _params.end()); +} + +const vespalib::string & +HttpRequest::get_param(const vespalib::string &name) const +{ + auto pos = _params.find(name); + if (pos == _params.end()) { + return _empty; + } + return pos->second; +} + } // namespace vespalib::portal diff --git a/vespalib/src/vespa/vespalib/portal/http_request.h b/vespalib/src/vespa/vespalib/portal/http_request.h index 51c7ab08da9..39467c3b248 100644 --- a/vespalib/src/vespa/vespalib/portal/http_request.h +++ b/vespalib/src/vespa/vespalib/portal/http_request.h @@ -14,6 +14,8 @@ private: // http stuff vespalib::string _method; vespalib::string _uri; + vespalib::string _path; + std::map<vespalib::string, vespalib::string> _params; vespalib::string _version; std::map<vespalib::string, vespalib::string> _headers; vespalib::string _host; @@ -43,6 +45,10 @@ public: const vespalib::string &get_header(const vespalib::string &name) const; const vespalib::string &get_host() const { return _host; } const vespalib::string &get_uri() const { return _uri; } + const vespalib::string &get_path() const { return _path; } + bool has_param(const vespalib::string &name) const; + const vespalib::string &get_param(const vespalib::string &name) const; + std::map<vespalib::string, vespalib::string> export_params() const { return _params; } }; } // namespace vespalib::portal diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp index 0d62d5728d1..ec2f1b78c03 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.cpp +++ b/vespalib/src/vespa/vespalib/portal/portal.cpp @@ -48,6 +48,34 @@ Portal::GetRequest::get_uri() const return _conn->get_request().get_uri(); } +const vespalib::string & +Portal::GetRequest::get_path() const +{ + assert(active()); + return _conn->get_request().get_path(); +} + +bool +Portal::GetRequest::has_param(const vespalib::string &name) const +{ + assert(active()); + return _conn->get_request().has_param(name); +} + +const vespalib::string & +Portal::GetRequest::get_param(const vespalib::string &name) const +{ + assert(active()); + return _conn->get_request().get_param(name); +} + +std::map<vespalib::string, vespalib::string> +Portal::GetRequest::export_params() const +{ + assert(active()); + return _conn->get_request().export_params(); +} + void Portal::GetRequest::respond_with_content(const vespalib::string &content_type, const vespalib::string &content) @@ -131,7 +159,7 @@ Portal::handle_http(portal::HttpConnection *conn) conn->respond_with_error(501, "Not Implemented"); } else { GetHandler *get_handler = nullptr; - auto guard = lookup_get_handler(conn->get_request().get_uri(), get_handler); + auto guard = lookup_get_handler(conn->get_request().get_path(), get_handler); if (guard.valid()) { assert(get_handler != nullptr); conn->resolve_host(_my_host); diff --git a/vespalib/src/vespa/vespalib/portal/portal.h b/vespalib/src/vespa/vespalib/portal/portal.h index 93424dda90c..fc3b81b37e6 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.h +++ b/vespalib/src/vespa/vespalib/portal/portal.h @@ -60,6 +60,10 @@ public: const vespalib::string &get_header(const vespalib::string &name) const; const vespalib::string &get_host() const; const vespalib::string &get_uri() const; + const vespalib::string &get_path() const; + bool has_param(const vespalib::string &name) const; + const vespalib::string &get_param(const vespalib::string &name) const; + std::map<vespalib::string, vespalib::string> export_params() const; void respond_with_content(const vespalib::string &content_type, const vespalib::string &content); void respond_with_error(int code, const vespalib::string &msg); diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp b/vsm/src/vespa/vsm/vsm/docsumconfig.cpp index 25c13967c49..ab89f4d460f 100644 --- a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumconfig.cpp @@ -20,7 +20,7 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & (overrideName == "absdist") || (overrideName == "subproject")) { - fieldWriter.reset(new EmptyDFW()); + fieldWriter = std::make_unique<EmptyDFW>(); rc = true; } else if ((overrideName == "attribute") || (overrideName == "attributecombiner") || |