diff options
42 files changed, 348 insertions, 233 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java index eb76446c045..caf5f0442eb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java @@ -2,16 +2,15 @@ package com.yahoo.searchdefinition.expressiontransforms; import com.yahoo.searchdefinition.FeatureNames; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.NameNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -21,8 +20,6 @@ import java.util.List; */ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfileTransformContext> { - public static final String CONSTANT = "constant"; - @Override public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { if (node instanceof ReferenceNode) { @@ -52,17 +49,18 @@ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfile } private ExpressionNode transformConstantReference(ReferenceNode node, RankProfileTransformContext context) { + Reference constantReference = FeatureNames.asConstantFeature(node.getName()); Value value = context.constants().get(node.getName()); if (value == null || value.type().rank() == 0) { + if (context.rankProfile().rankingConstants().get(node.getName()) != null) // Large constants: Transform reference but don't add value + return new ReferenceNode(constantReference); return node; } TensorValue tensorValue = (TensorValue)value; - String featureName = CONSTANT + "(" + node.getName() + ")"; String tensorType = tensorValue.asTensor().type().toString(); - context.rankProperties().put(featureName + ".value", tensorValue.toString()); - context.rankProperties().put(featureName + ".type", tensorType); - // TODO: This allows us to reference constant "a" as "a" instead of "constant(a)", but we shouldn't allow that - return new ReferenceNode(CONSTANT, Arrays.asList(new NameNode(node.getName())), null); + context.rankProperties().put(constantReference.toString() + ".value", tensorValue.toString()); + context.rankProperties().put(constantReference.toString() + ".type", tensorType); + return new ReferenceNode(constantReference); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index 434cae04747..2c0e1eaa56a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -3,7 +3,9 @@ package com.yahoo.searchdefinition.expressiontransforms; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.searchdefinition.MapEvaluationTypeContext; import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.transform.TransformContext; @@ -22,6 +24,7 @@ public class RankProfileTransformContext extends TransformContext { private final ImportedMlModels importedModels; private final Map<String, RankProfile.RankingExpressionFunction> inlineFunctions; private final Map<String, String> rankProperties = new HashMap<>(); + private final MapEvaluationTypeContext types; public RankProfileTransformContext(RankProfile rankProfile, QueryProfileRegistry queryProfiles, @@ -33,6 +36,7 @@ public class RankProfileTransformContext extends TransformContext { this.queryProfiles = queryProfiles; this.importedModels = importedModels; this.inlineFunctions = inlineFunctions; + this.types = rankProfile.typeContext(queryProfiles); } public RankProfile rankProfile() { return rankProfile; } @@ -41,4 +45,10 @@ public class RankProfileTransformContext extends TransformContext { public Map<String, RankProfile.RankingExpressionFunction> inlineFunctions() { return inlineFunctions; } public Map<String, String> rankProperties() { return rankProperties; } + /** + * Returns the types known in this context. We may have type information for references + * for which no value is available + */ + public MapEvaluationTypeContext types() { return types; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java index fe232299363..5d03c323803 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java @@ -69,7 +69,7 @@ public class TensorTransformer extends ExpressionTransformer<RankProfileTransfor ExpressionNode arg1 = node.children().get(0); Optional<String> dimension = dimensionName(node.children().get(1)); if (dimension.isPresent()) { - TensorType type = arg1.type(context.rankProfile().typeContext(context.queryProfiles())); + TensorType type = arg1.type(context.types()); if (type.dimension(dimension.get()).isPresent()) { return replaceMaxAndMinFunction(node); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index e94fa9bf040..9ea51a83bf1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -15,6 +15,7 @@ public class LogserverContainer extends Container { public LogserverContainer(AbstractConfigProducer parent) { super(parent, "" + 0, 0); addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, ((LogserverContainerCluster) parent).getName(), true)); + appendJvmOptions("-Xms32m -Xmx512m"); } @Override diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java index a75699d2a1d..34b727f9f4e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java @@ -30,7 +30,7 @@ public class VespaMlModelTestCase { "rankingExpression(foo1).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1), max, x) * 3.0\n" + "rankingExpression(foo1).input2.type : tensor(x[3])\n" + "rankingExpression(foo1).input1.type : tensor(name{},x[3])\n" + - "rankingExpression(foo2).rankingScript : max(reduce(input1 * input2, sum, name) * constant1asLarge,x) * 3.0\n" + + "rankingExpression(foo2).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1asLarge), max, x) * 3.0\n" + "rankingExpression(foo2).input2.type : tensor(x[3])\n" + "rankingExpression(foo2).input1.type : tensor(name{},x[3])\n"; diff --git a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java index 5586fd2f996..62ee16993fd 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java @@ -35,11 +35,11 @@ import java.util.concurrent.Executor; import java.util.stream.Collectors; /** - * XML rendering of search results. This is NOT the default (but it once was). + * XML rendering of search results. * * @author Tony Vaagenes */ -@SuppressWarnings({ "rawtypes", "deprecation" }) +@SuppressWarnings("rawtypes") public final class XmlRenderer extends AsynchronousSectionedRenderer<Result> { public static final String DEFAULT_MIMETYPE = "text/xml"; diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 5fc9ee0b2f1..eafc6d214d3 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -32,10 +32,10 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.objects.FieldBase; -import org.apache.commons.codec.binary.Base64; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Base64; import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -45,7 +45,7 @@ import java.util.Set; * @author Vegard Sjonfjell */ public class JsonSerializationHelper { - private final static Base64 base64Encoder = new Base64(); + private final static Base64.Encoder base64Encoder = Base64.getEncoder(); // Important: _basic_ format static class JsonSerializationException extends RuntimeException { public JsonSerializationException(Exception base) { diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index 454ad72f344..57946741335 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -485,7 +485,7 @@ public class DocumentUpdateJsonSerializerTest { " 'update': 'DOCUMENT_ID',", " 'fields': {", " 'raw_field': {", - " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw==\\r\\n'", + " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw=='", " }", " }", "}" diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 9df7d1f91c1..f8ee23e86ba 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -53,7 +53,6 @@ import com.yahoo.tensor.MappedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; -import org.apache.commons.codec.binary.Base64; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -64,8 +63,8 @@ import org.mockito.internal.matchers.Contains; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -603,10 +602,24 @@ public class JsonReaderTestCase { @Test public void testRaw() throws IOException { - String stuff = new String(new JsonStringEncoder().quoteAsString(new Base64().encodeToString(Utf8.toBytes("smoketest")))); + String base64 = new String(new JsonStringEncoder().quoteAsString( + Base64.getEncoder().encodeToString(Utf8.toBytes("smoketest")))); + String s = fieldStringFromBase64RawContent(base64); + assertEquals("smoketest", s); + } + + @Test + public void can_read_legacy_chunked_base64_raw_field_encoding() throws IOException { + String expected = "this is a string with an impressive length. it's long enough to reach the end of the line, wow!"; + String base64withDelims = "dGhpcyBpcyBhIHN0cmluZyB3aXRoIGFuIGltcHJlc3NpdmUgbGVuZ3RoLiBpdCdzIGxvbmcgZW5v\\r\\n" + + "dWdoIHRvIHJlYWNoIHRoZSBlbmQgb2YgdGhlIGxpbmUsIHdvdyE=\\r\\n"; + assertEquals(expected, fieldStringFromBase64RawContent(base64withDelims)); + } + + private String fieldStringFromBase64RawContent(String base64data) throws IOException { JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testraw::whee',", " 'fields': {", - " 'actualraw': '" + stuff + "' }}")); + " 'actualraw': '" + base64data + "' }}")); DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); @@ -615,8 +628,7 @@ public class JsonReaderTestCase { FieldValue f = doc.getFieldValue(doc.getField("actualraw")); assertSame(Raw.class, f.getClass()); Raw s = (Raw) f; - ByteBuffer b = s.getByteBuffer(); - assertEquals("smoketest", Utf8.toString(b)); + return Utf8.toString(s.getByteBuffer()); } @Test diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 0ab00b4e2bc..fcc67136526 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -26,7 +26,6 @@ import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.json.readers.VespaJsonDocumentReader; import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; -import org.apache.commons.codec.binary.Base64; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -273,10 +272,13 @@ public class JsonWriterTestCase { } @Test - public final void rawTest() throws IOException { + public final void raw_fields_are_emitted_as_basic_base64() throws IOException { + // "string long enough to emit more than 76 base64 characters and which should certainly not be newline-delimited!" String payload = new String( - new JsonStringEncoder().quoteAsString(new Base64() - .encodeToString(Utf8.toBytes("smoketest")))); + new JsonStringEncoder().quoteAsString( + "c3RyaW5nIGxvbmcgZW5vdWdoIHRvIGVtaXQgbW9yZSB0aGFuIDc2IGJhc2U2NCBjaGFyYWN0ZXJzIGFuZC" + + "B3aGljaCBzaG91bGQgY2VydGFpbmx5IG5vdCBiZSBuZXdsaW5lLWRlbGltaXRlZCE=")); + String docId = "id:unittest:testraw::whee"; String fields = "{ \"actualraw\": \"" + payload + "\"" + " }"; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ImportedModel.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ImportedModel.java index 58962d1a5ff..0c570261ae7 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ImportedModel.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ImportedModel.java @@ -147,9 +147,6 @@ public class ImportedModel implements ImportedMlModel { */ public File relativeFile(String relativePath, String descriptionOfPath) { File file = new File(new File(source()).getParent(), relativePath); - if (file.isAbsolute()) - throw new IllegalArgumentException(descriptionOfPath + " uses the absolute file path '" + relativePath + - "'. File paths must be relative to the directory referencing them"); if ( ! file.exists()) throw new IllegalArgumentException(descriptionOfPath + " references '" + relativePath + "', but this file does not exist"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java index 4770b3a3226..5317aa74737 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java @@ -28,7 +28,6 @@ public class NodeAttributes { private Optional<Version> vespaVersion = Optional.empty(); private Optional<Version> currentOsVersion = Optional.empty(); private Optional<Instant> currentFirmwareCheck = Optional.empty(); - private Optional<String> hardwareFailureDescription = Optional.empty(); private Optional<Boolean> wantToDeprovision = Optional.empty(); /** The list of reports to patch. A null value is used to remove the report. */ private Map<String, JsonNode> reports = new TreeMap<>(); @@ -69,10 +68,6 @@ public class NodeAttributes { return this; } - public NodeAttributes withHardwareFailureDescription(String hardwareFailureDescription) { - this.hardwareFailureDescription = Optional.of(hardwareFailureDescription); - return this; - } public NodeAttributes withWantToDeprovision(boolean wantToDeprovision) { this.wantToDeprovision = Optional.of(wantToDeprovision); @@ -118,10 +113,6 @@ public class NodeAttributes { return currentFirmwareCheck; } - public Optional<String> getHardwareFailureDescription() { - return hardwareFailureDescription; - } - public Optional<Boolean> getWantToDeprovision() { return wantToDeprovision; } @@ -133,7 +124,7 @@ public class NodeAttributes { @Override public int hashCode() { return Objects.hash(restartGeneration, rebootGeneration, dockerImage, vespaVersion, currentOsVersion, - currentFirmwareCheck, hardwareFailureDescription, wantToDeprovision, reports); + currentFirmwareCheck, wantToDeprovision, reports); } public boolean isEmpty() { @@ -153,7 +144,6 @@ public class NodeAttributes { && Objects.equals(vespaVersion, other.vespaVersion) && Objects.equals(currentOsVersion, other.currentOsVersion) && Objects.equals(currentFirmwareCheck, other.currentFirmwareCheck) - && Objects.equals(hardwareFailureDescription, other.hardwareFailureDescription) && Objects.equals(reports, other.reports) && Objects.equals(wantToDeprovision, other.wantToDeprovision); } @@ -167,7 +157,6 @@ public class NodeAttributes { vespaVersion.map(ver -> "vespaVersion=" + ver.toFullString()), currentOsVersion.map(ver -> "currentOsVersion=" + ver.toFullString()), currentFirmwareCheck.map(at -> "currentFirmwareCheck=" + at), - hardwareFailureDescription.map(hwDesc -> "hardwareFailureDescription=" + hwDesc), Optional.ofNullable(reports.isEmpty() ? null : "reports=" + reports), wantToDeprovision.map(depr -> "wantToDeprovision=" + depr)) .filter(Optional::isPresent) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index cdc90e214bd..d402e75ff7b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -55,7 +55,6 @@ public class NodeSpec { private final Set<String> ipAddresses; private final Set<String> additionalIpAddresses; - private final Optional<String> hardwareFailureDescription; private final NodeReports reports; private final Optional<String> parentHostname; @@ -90,7 +89,6 @@ public class NodeSpec { double bandwidth, Set<String> ipAddresses, Set<String> additionalIpAddresses, - Optional<String> hardwareFailureDescription, NodeReports reports, Optional<String> parentHostname) { this.hostname = Objects.requireNonNull(hostname); @@ -122,7 +120,6 @@ public class NodeSpec { this.bandwidth = bandwidth; this.ipAddresses = Objects.requireNonNull(ipAddresses); this.additionalIpAddresses = Objects.requireNonNull(additionalIpAddresses); - this.hardwareFailureDescription = Objects.requireNonNull(hardwareFailureDescription); this.reports = Objects.requireNonNull(reports); this.parentHostname = Objects.requireNonNull(parentHostname); } @@ -243,10 +240,6 @@ public class NodeSpec { return additionalIpAddresses; } - public Optional<String> getHardwareFailureDescription() { - return hardwareFailureDescription; - } - public NodeReports getReports() { return reports; } public Optional<String> getParentHostname() { @@ -288,7 +281,6 @@ public class NodeSpec { Objects.equals(bandwidth, that.bandwidth) && Objects.equals(ipAddresses, that.ipAddresses) && Objects.equals(additionalIpAddresses, that.additionalIpAddresses) && - Objects.equals(hardwareFailureDescription, that.hardwareFailureDescription) && Objects.equals(reports, that.reports) && Objects.equals(parentHostname, that.parentHostname); } @@ -324,7 +316,6 @@ public class NodeSpec { bandwidth, ipAddresses, additionalIpAddresses, - hardwareFailureDescription, reports, parentHostname); } @@ -360,7 +351,6 @@ public class NodeSpec { + " bandwidth=" + bandwidth + " ipAddresses=" + ipAddresses + " additionalIpAddresses=" + additionalIpAddresses - + " hardwareFailureDescription=" + hardwareFailureDescription + " reports=" + reports + " parentHostname=" + parentHostname + " }"; @@ -396,7 +386,6 @@ public class NodeSpec { private double bandwidth; private Set<String> ipAddresses = Set.of(); private Set<String> additionalIpAddresses = Set.of(); - private Optional<String> hardwareFailureDescription = Optional.empty(); private NodeReports reports = new NodeReports(); private Optional<String> parentHostname = Optional.empty(); @@ -433,7 +422,6 @@ public class NodeSpec { node.currentRestartGeneration.ifPresent(this::currentRestartGeneration); node.wantedFirmwareCheck.ifPresent(this::wantedFirmwareCheck); node.currentFirmwareCheck.ifPresent(this::currentFirmwareCheck); - node.hardwareFailureDescription.ifPresent(this::hardwareFailureDescription); node.parentHostname.ifPresent(this::parentHostname); } @@ -577,11 +565,6 @@ public class NodeSpec { return this; } - public Builder hardwareFailureDescription(String hardwareFailureDescription) { - this.hardwareFailureDescription = Optional.of(hardwareFailureDescription); - return this; - } - public Builder reports(NodeReports reports) { this.reports = reports; return this; @@ -607,7 +590,6 @@ public class NodeSpec { attributes.getCurrentOsVersion().ifPresent(this::currentOsVersion); attributes.getRebootGeneration().ifPresent(this::currentRebootGeneration); attributes.getRestartGeneration().ifPresent(this::currentRestartGeneration); - attributes.getHardwareFailureDescription().ifPresent(this::hardwareFailureDescription); attributes.getWantToDeprovision().ifPresent(this::wantToDeprovision); NodeReports.fromMap(attributes.getReports()); @@ -718,10 +700,6 @@ public class NodeSpec { return additionalIpAddresses; } - public Optional<String> getHardwareFailureDescription() { - return hardwareFailureDescription; - } - public NodeReports getReports() { return reports; } @@ -739,7 +717,7 @@ public class NodeSpec { wantedRebootGeneration, currentRebootGeneration, wantedFirmwareCheck, currentFirmwareCheck, modelName, minCpuCores, minMainMemoryAvailableGb, minDiskAvailableGb, - fastDisk, bandwidth, ipAddresses, additionalIpAddresses, hardwareFailureDescription, + fastDisk, bandwidth, ipAddresses, additionalIpAddresses, reports, parentHostname); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index ab14b8cba20..ca52eca13d2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -201,7 +201,6 @@ public class RealNodeRepository implements NodeRepository { node.bandwidth, node.ipAddresses, node.additionalIpAddresses, - Optional.ofNullable(node.hardwareFailureDescription), reports, Optional.ofNullable(node.parentHostname)); } @@ -226,7 +225,6 @@ public class RealNodeRepository implements NodeRepository { node.vespaVersion = nodeAttributes.getVespaVersion().map(Version::toFullString).orElse(null); node.currentOsVersion = nodeAttributes.getCurrentOsVersion().map(Version::toFullString).orElse(null); node.currentFirmwareCheck = nodeAttributes.getCurrentFirmwareCheck().map(Instant::toEpochMilli).orElse(null); - node.hardwareFailureDescription = nodeAttributes.getHardwareFailureDescription().orElse(null); node.wantToDeprovision = nodeAttributes.getWantToDeprovision().orElse(null); Map<String, JsonNode> reports = nodeAttributes.getReports(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index a4cf61ef88f..2b4f9277fc7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -62,10 +62,6 @@ public class NodeRepositoryNode { public Boolean fastDisk; @JsonProperty("bandwidth") public Double bandwidth; - @JsonProperty("hardwareFailure") - public Boolean hardwareFailure; - @JsonProperty("hardwareFailureDescription") - public String hardwareFailureDescription; @JsonProperty("environment") public String environment; @JsonProperty("type") @@ -118,8 +114,6 @@ public class NodeRepositoryNode { ", failCount=" + failCount + ", fastDisk=" + fastDisk + ", bandwidth=" + bandwidth + - ", hardwareFailure=" + hardwareFailure + - ", hardwareFailureDescription='" + hardwareFailureDescription + '\'' + ", environment='" + environment + '\'' + ", type='" + type + '\'' + ", wantedDockerImage='" + wantedDockerImage + '\'' + diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index d0b19d77181..9a343667fd6 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -1,27 +1,29 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("feed_and_search_test"); #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/fieldvalue.h> -#include <vespa/searchlib/memoryindex/memory_index.h> +#include <vespa/searchlib/common/documentsummary.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/diskindex/diskindex.h> +#include <vespa/searchlib/diskindex/fusion.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/memoryindex/memory_index.h> +#include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/searchlib/query/base.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/blueprint.h> -#include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> +#include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <sstream> -#include <vespa/searchlib/diskindex/fusion.h> -#include <vespa/searchlib/common/documentsummary.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> + +#include <vespa/log/log.h> +LOG_SETUP("feed_and_search_test"); using document::DataType; using document::Document; @@ -32,25 +34,26 @@ using search::TuneFileSearch; using search::diskindex::DiskIndex; using search::diskindex::IndexBuilder; using search::diskindex::SelectorArray; +using search::docsummary::DocumentSummary; using search::fef::FieldPositionsIterator; using search::fef::MatchData; using search::fef::MatchDataLayout; using search::fef::TermFieldHandle; using search::fef::TermFieldMatchData; using search::index::DocBuilder; -using search::index::Schema; using search::index::DummyFileHeaderContext; +using search::index::Schema; +using search::index::test::MockFieldLengthInspector; using search::memoryindex::MemoryIndex; using search::query::SimpleStringTerm; using search::queryeval::Blueprint; +using search::queryeval::FakeRequestContext; using search::queryeval::FieldSpec; using search::queryeval::FieldSpecList; using search::queryeval::SearchIterator; using search::queryeval::Searchable; -using search::queryeval::FakeRequestContext; using std::ostringstream; using vespalib::string; -using search::docsummary::DocumentSummary; namespace { @@ -148,7 +151,7 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { vespalib::ThreadStackExecutor sharedExecutor(2, 0x10000); search::SequencedTaskExecutor indexFieldInverter(2); search::SequencedTaskExecutor indexFieldWriter(2); - MemoryIndex memory_index(schema, indexFieldInverter, indexFieldWriter); + MemoryIndex memory_index(schema, MockFieldLengthInspector(), indexFieldInverter, indexFieldWriter); DocBuilder doc_builder(schema); Document::UP doc = buildDocument(doc_builder, doc_id1, word1); diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index 8f9944e178e..25e7a4ffa6b 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -1,19 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Unit tests for fusionrunner. +#include <vespa/fastos/file.h> #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/searchcorespi/index/fusionrunner.h> -#include <vespa/searchlib/memoryindex/memory_index.h> +#include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> -#include <vespa/searchlib/common/isequencedtaskexecutor.h> +#include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/fastos/file.h> #include <set> using document::Document; @@ -35,6 +35,7 @@ using search::index::DocBuilder; using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::schema::DataType; +using search::index::test::MockFieldLengthInspector; using search::memoryindex::MemoryIndex; using search::query::SimpleStringTerm; using search::queryeval::Blueprint; @@ -171,7 +172,8 @@ void Test::createIndex(const string &dir, uint32_t id, bool fusion) { Schema schema = getSchema(); DocBuilder doc_builder(schema); - MemoryIndex memory_index(schema, _threadingService.indexFieldInverter(), + MemoryIndex memory_index(schema, MockFieldLengthInspector(), + _threadingService.indexFieldInverter(), _threadingService.indexFieldWriter()); addDocument(doc_builder, memory_index, *_selector, id, id + 0, term); addDocument(doc_builder, memory_index, *_selector, id, id + 1, "bar"); diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 9ceb85981c0..4149d563bf9 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -19,6 +19,7 @@ #include <vespa/searchlib/memoryindex/field_index_collection.h> #include <vespa/searchlib/memoryindex/field_inverter.h> #include <vespa/searchlib/queryeval/isourceselector.h> +#include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> @@ -41,6 +42,7 @@ using search::index::DocBuilder; using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::schema::DataType; +using search::index::test::MockFieldLengthInspector; using vespalib::makeLambdaTask; using search::memoryindex::CompactWordsStore; using search::memoryindex::FieldIndexCollection; @@ -58,8 +60,7 @@ using namespace searchcorespi::index; namespace { -class IndexManagerDummyReconfigurer : public searchcorespi::IIndexManager::Reconfigurer -{ +class IndexManagerDummyReconfigurer : public searchcorespi::IIndexManager::Reconfigurer { virtual bool reconfigure(vespalib::Closure0<bool>::UP closure) override { @@ -360,7 +361,7 @@ TEST_F("requireThatSourceSelectorIsFlushed", Fixture) { TEST_F("requireThatFlushStatsAreCalculated", Fixture) { Schema schema(getSchema()); - FieldIndexCollection fic(schema); + FieldIndexCollection fic(schema, MockFieldLengthInspector()); SequencedTaskExecutor invertThreads(2); SequencedTaskExecutor pushThreads(2); search::memoryindex::DocumentInverter inverter(schema, invertThreads, diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index b74cc2c603f..8e838414015 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -37,9 +37,11 @@ IndexManager::MaintainerOperations::MaintainerOperations(const FileHeaderContext } IMemoryIndex::SP -IndexManager::MaintainerOperations::createMemoryIndex(const Schema &schema, SerialNum serialNum) +IndexManager::MaintainerOperations::createMemoryIndex(const Schema& schema, + const IFieldLengthInspector& inspector, + SerialNum serialNum) { - return std::make_shared<MemoryIndexWrapper>(schema, _fileHeaderContext, _tuneFileIndexing, + return std::make_shared<MemoryIndexWrapper>(schema, inspector, _fileHeaderContext, _tuneFileIndexing, _threadingService, serialNum); } diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 4f2d03c81e6..b14912239a3 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -48,7 +48,9 @@ public: size_t cacheSize, searchcorespi::index::IThreadingService &threadingService); - IMemoryIndex::SP createMemoryIndex(const Schema &schema, SerialNum serialNum) override; + IMemoryIndex::SP createMemoryIndex(const Schema& schema, + const IFieldLengthInspector& inspector, + SerialNum serialNum) override; IDiskIndex::SP loadDiskIndex(const vespalib::string &indexDir) override; IDiskIndex::SP reloadDiskIndex(const IDiskIndex &oldIndex) override; bool runFusion(const Schema &schema, const vespalib::string &outputDir, diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp index 3d1e04196a6..d206388ca04 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp @@ -17,13 +17,13 @@ using vespalib::IllegalStateException; namespace proton { -MemoryIndexWrapper::MemoryIndexWrapper(const search::index::Schema &schema, - const search::common::FileHeaderContext &fileHeaderContext, - const TuneFileIndexing &tuneFileIndexing, - searchcorespi::index::IThreadingService & - threadingService, +MemoryIndexWrapper::MemoryIndexWrapper(const search::index::Schema& schema, + const search::index::IFieldLengthInspector& inspector, + const search::common::FileHeaderContext& fileHeaderContext, + const TuneFileIndexing& tuneFileIndexing, + searchcorespi::index::IThreadingService& threadingService, search::SerialNum serialNum) - : _index(schema, threadingService.indexFieldInverter(), + : _index(schema, inspector, threadingService.indexFieldInverter(), threadingService.indexFieldWriter()), _serialNum(serialNum), _fileHeaderContext(fileHeaderContext), @@ -62,9 +62,7 @@ MemoryIndexWrapper::accept(searchcorespi::IndexSearchableVisitor &visitor) const FieldLengthInfo MemoryIndexWrapper::get_field_length_info(const vespalib::string& field_name) const { - // TODO: implement - (void) field_name; - return FieldLengthInfo(); + return _index.get_field_length_info(field_name); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index d94a259eb24..dfebaff20f3 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -24,11 +24,11 @@ private: const search::TuneFileIndexing _tuneFileIndexing; public: - MemoryIndexWrapper(const search::index::Schema &schema, - const search::common::FileHeaderContext &fileHeaderContext, - const search::TuneFileIndexing &tuneFileIndexing, - searchcorespi::index::IThreadingService & - threadingService, + MemoryIndexWrapper(const search::index::Schema& schema, + const search::index::IFieldLengthInspector& inspector, + const search::common::FileHeaderContext& fileHeaderContext, + const search::TuneFileIndexing& tuneFileIndexing, + searchcorespi::index::IThreadingService& threadingService, SerialNum serialNum); /** diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmaintaineroperations.h b/searchcorespi/src/vespa/searchcorespi/index/iindexmaintaineroperations.h index 507ccf9483b..99f17b12b79 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmaintaineroperations.h +++ b/searchcorespi/src/vespa/searchcorespi/index/iindexmaintaineroperations.h @@ -6,6 +6,7 @@ #include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/searchlib/diskindex/docidmapper.h> +#include <vespa/searchlib/index/i_field_length_inspector.h> namespace searchcorespi::index { @@ -13,6 +14,7 @@ namespace searchcorespi::index { * Interface for operations needed by an index maintainer. */ struct IIndexMaintainerOperations { + using IFieldLengthInspector = search::index::IFieldLengthInspector; using Schema = search::index::Schema; using SelectorArray = search::diskindex::SelectorArray; virtual ~IIndexMaintainerOperations() {} @@ -20,7 +22,9 @@ struct IIndexMaintainerOperations { /** * Creates a new memory index using the given schema. */ - virtual IMemoryIndex::SP createMemoryIndex(const Schema &schema, search::SerialNum serialNum) = 0; + virtual IMemoryIndex::SP createMemoryIndex(const Schema& schema, + const IFieldLengthInspector& inspector, + search::SerialNum serialNum) = 0; /** * Loads a disk index from the given directory. diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index acde26ad554..a174592eb55 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -870,11 +870,11 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, _selector.reset(getSourceSelector().cloneAndSubtract(ost.str(), id_diff).release()); assert(_last_fusion_id == _selector->getBaseId()); } - _current_index = operations.createMemoryIndex(_schema, _current_serial_num); _current_index_id = getNewAbsoluteId() - _last_fusion_id; assert(_current_index_id < ISourceSelector::SOURCE_LIMIT); _selector->setDefaultSource(_current_index_id); ISearchableIndexCollection::UP sourceList(loadDiskIndexes(spec, ISearchableIndexCollection::UP(new IndexCollection(_selector)))); + _current_index = operations.createMemoryIndex(_schema, *sourceList, _current_serial_num); LOG(debug, "Index manager created with flushed serial num %" PRIu64, _flush_serial_num); sourceList->append(_current_index_id, _current_index); sourceList->setCurrentIndex(_current_index_id); @@ -900,7 +900,7 @@ IndexMaintainer::initFlush(SerialNum serialNum, searchcorespi::FlushStats * stat _current_serial_num = std::max(_current_serial_num, serialNum); } - IMemoryIndex::SP new_index(_operations.createMemoryIndex(getSchema(), _current_serial_num)); + IMemoryIndex::SP new_index(_operations.createMemoryIndex(getSchema(), *_current_index, _current_serial_num)); FlushArgs args; args.stats = stats; scheduleCommit(); @@ -1208,7 +1208,7 @@ IndexMaintainer::setSchema(const Schema & schema, SerialNum serialNum) { assert(_ctx.getThreadingService().master().isCurrentThread()); pruneRemovedFields(schema, serialNum); - IMemoryIndex::SP new_index(_operations.createMemoryIndex(schema, _current_serial_num)); + IMemoryIndex::SP new_index(_operations.createMemoryIndex(schema, *_current_index, _current_serial_num)); SetSchemaArgs args; args._newSchema = schema; diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 2ef8448db8b..1571cef630b 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -1,28 +1,31 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memory_index.h> + +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/searchlib/common/scheduletaskcallback.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/index/i_field_length_inspector.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> +#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> -#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/searchlib/test/index/mock_field_length_inspector.h> +#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> -#include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/document/repo/configbuilder.h> -#include <vespa/document/repo/fixedtyperepo.h> -#include <vespa/document/annotation/spanlist.h> -#include <vespa/document/annotation/spantree.h> -#include <vespa/searchlib/util/rand48.h> #include <vespa/log/log.h> LOG_SETUP("memoryindexstress_test"); @@ -36,17 +39,18 @@ using document::FieldValue; using document::Span; using document::SpanList; using document::StringFieldValue; +using namespace search::fef; +using namespace search::index; +using namespace search::memoryindex; +using namespace search::queryeval; using search::ScheduleTaskCallback; using search::index::schema::DataType; -using vespalib::makeLambdaTask; using search::query::Node; using search::query::SimplePhrase; using search::query::SimpleStringTerm; -using namespace search::fef; -using namespace search::index; -using namespace search::memoryindex; -using namespace search::queryeval; +using search::index::test::MockFieldLengthInspector; using vespalib::asciistream; +using vespalib::makeLambdaTask; namespace { @@ -189,8 +193,6 @@ Node::UP makePhrase(const std::string &term1, const std::string &term2) { } // namespace - - struct Fixture { Schema schema; DocumentTypeRepo repo; @@ -249,7 +251,7 @@ Fixture::Fixture(uint32_t readThreads) _executor(1, 128 * 1024), _invertThreads(2), _pushThreads(2), - index(schema, _invertThreads, _pushThreads), + index(schema, MockFieldLengthInspector(), _invertThreads, _pushThreads), _readThreads(readThreads), _writer(1, 128 * 1024), _readers(readThreads, 128 * 1024), diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index 339e196c9bf..694af2f1ad1 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -1,8 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/btree/btreenode.hpp> -#include <vespa/vespalib/btree/btreenodeallocator.hpp> -#include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/fusion.h> @@ -15,9 +12,13 @@ #include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> #include <vespa/searchlib/memoryindex/posting_iterator.h> +#include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/searchlib/util/filekit.h> -#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/btree/btreenode.hpp> +#include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> LOG_SETUP("fusion_test"); @@ -28,12 +29,13 @@ using document::Document; using fef::FieldPositionsIterator; using fef::TermFieldMatchData; using fef::TermFieldMatchDataArray; -using memoryindex::FieldIndexCollection; using memoryindex::DocumentInverter; +using memoryindex::FieldIndexCollection; using queryeval::SearchIterator; using search::common::FileHeaderContext; using search::index::schema::CollectionType; using search::index::schema::DataType; +using search::index::test::MockFieldLengthInspector; using namespace index; @@ -268,7 +270,7 @@ Test::requireThatFusionIsWorking(const vespalib::string &prefix, bool directio, addField("f0").addField("f1"). addField("f2").addField("f3"). addField("f4")); - FieldIndexCollection fic(schema); + FieldIndexCollection fic(schema, MockFieldLengthInspector()); DocBuilder b(schema); SequencedTaskExecutor invertThreads(2); SequencedTaskExecutor pushThreads(2); diff --git a/searchlib/src/tests/index/field_length_calculator/field_length_calculator_test.cpp b/searchlib/src/tests/index/field_length_calculator/field_length_calculator_test.cpp index c99d241cbc0..8ec1655e372 100644 --- a/searchlib/src/tests/index/field_length_calculator/field_length_calculator_test.cpp +++ b/searchlib/src/tests/index/field_length_calculator/field_length_calculator_test.cpp @@ -63,6 +63,14 @@ TEST(FieldLengthCalculatorTest, average_until_max_num_samples) EXPECT_EQ(max_num_samples, calc.get_num_samples()); } +TEST(FieldLengthCalculatorTest, calculator_can_return_info_object) +{ + FieldLengthCalculator calc(3, 5); + auto info = calc.get_info(); + EXPECT_EQ(3, info.get_average_field_length()); + EXPECT_EQ(5, info.get_num_samples()); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 95861643f84..234cf9b5e84 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/btree/btreenodeallocator.hpp> -#include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/diskindex/fusion.h> #include <vespa/searchlib/diskindex/indexbuilder.h> @@ -16,7 +14,10 @@ #include <vespa/searchlib/memoryindex/field_inverter.h> #include <vespa/searchlib/memoryindex/ordered_field_index_inserter.h> #include <vespa/searchlib/memoryindex/posting_iterator.h> +#include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/searchlib/test/memoryindex/wrap_inserter.h> +#include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> @@ -33,6 +34,7 @@ using document::Document; using queryeval::SearchIterator; using search::index::schema::CollectionType; using search::index::schema::DataType; +using search::index::test::MockFieldLengthInspector; using vespalib::GenerationHandler; namespace memoryindex { @@ -294,7 +296,7 @@ public: MyInserter(const Schema &schema) : _wordStoreScan(), _mock(), - _fieldIndexes(schema), + _fieldIndexes(schema, MockFieldLengthInspector()), _features(), _inserter(nullptr) { @@ -490,7 +492,7 @@ struct FieldIndexCollectionTest : public ::testing::Test { FieldIndexCollection fic; FieldIndexCollectionTest() : schema(make_multi_field_schema()), - fic(schema) + fic(schema, MockFieldLengthInspector()) { } ~FieldIndexCollectionTest() {} @@ -755,7 +757,7 @@ public: InverterTest(const Schema& schema) : _schema(schema), - _fic(_schema), + _fic(_schema, MockFieldLengthInspector()), _b(_schema), _invertThreads(2), _pushThreads(2), diff --git a/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt index d6b6691f05d..0dd42eacf30 100644 --- a/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchlib_memory_index_test_app TEST memory_index_test.cpp DEPENDS searchlib + gtest ) vespa_add_test(NAME searchlib_memory_index_test_app COMMAND searchlib_memory_index_test_app) diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 50f44074683..dd4bb2cef7f 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> - #include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchlib/index/i_field_length_inspector.h> #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> @@ -15,6 +14,7 @@ #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> #include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/threadstackexecutor.h> @@ -25,6 +25,8 @@ using document::Document; using document::FieldValue; using search::ScheduleTaskCallback; using search::index::schema::DataType; +using search::index::FieldLengthInfo; +using search::index::IFieldLengthInspector; using vespalib::makeLambdaTask; using search::query::Node; using search::query::SimplePhrase; @@ -36,12 +38,25 @@ using namespace search::queryeval; //----------------------------------------------------------------------------- -struct Setup { +struct MySetup : public IFieldLengthInspector { Schema schema; - Setup &field(const std::string &name) { + std::map<vespalib::string, FieldLengthInfo> field_lengths; + MySetup &field(const std::string &name) { schema.addIndexField(Schema::IndexField(name, DataType::STRING)); return *this; } + MySetup& field_length(const vespalib::string& field_name, const FieldLengthInfo& info) { + field_lengths[field_name] = info; + return *this; + } + FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { + auto itr = field_lengths.find(field_name); + if (itr != field_lengths.end()) { + return itr->second; + } + return FieldLengthInfo(); + } + }; //----------------------------------------------------------------------------- @@ -56,7 +71,7 @@ struct Index { uint32_t docid; std::string currentField; - Index(const Setup &setup); + Index(const MySetup &setup); ~Index(); void closeField() { if (!currentField.empty()) { @@ -105,12 +120,12 @@ private: }; -Index::Index(const Setup &setup) +Index::Index(const MySetup &setup) : schema(setup.schema), _executor(1, 128 * 1024), _invertThreads(2), _pushThreads(2), - index(schema, _invertThreads, _pushThreads), + index(schema, setup, _invertThreads, _pushThreads), builder(schema), docid(1), currentField() @@ -158,15 +173,19 @@ verifyResult(const FakeResult &expect, fields.add(field); Blueprint::UP result = index.createBlueprint(requestContext, fields, term); - if (!EXPECT_TRUE(result.get() != 0)) { + bool valid_result = result.get() != 0; + EXPECT_TRUE(valid_result); + if (!valid_result) { return false; } - EXPECT_EQUAL(expect.inspect().size(), result->getState().estimate().estHits); - EXPECT_EQUAL(expect.inspect().empty(), result->getState().estimate().empty); + EXPECT_EQ(expect.inspect().size(), result->getState().estimate().estHits); + EXPECT_EQ(expect.inspect().empty(), result->getState().estimate().empty); result->fetchPostings(true); SearchIterator::UP search = result->createSearch(*match_data, true); - if (!EXPECT_TRUE(search.get() != 0)) { + bool valid_search = search.get() != 0; + EXPECT_TRUE(valid_search); + if (!valid_search) { return false; } TermFieldMatchData &tmd = *match_data->resolveTermField(handle); @@ -176,14 +195,15 @@ verifyResult(const FakeResult &expect, for (search->seek(1); !search->isAtEnd(); search->seek(search->getDocId() + 1)) { actual.doc(search->getDocId()); search->unpack(search->getDocId()); - EXPECT_EQUAL(search->getDocId(), tmd.getDocId()); + EXPECT_EQ(search->getDocId(), tmd.getDocId()); FieldPositionsIterator p = tmd.getIterator(); actual.len(p.getFieldLength()); for (; p.valid(); p.next()) { actual.pos(p.getPosition()); } } - return EXPECT_EQUAL(expect, actual); + EXPECT_EQ(expect, actual); + return expect == actual; } namespace { @@ -202,9 +222,9 @@ Node::UP makePhrase(const std::string &term1, const std::string &term2) { // tests basic usage; index some documents in docid order and perform // some searches. -TEST("testIndexAndSearch") +TEST(MemoryIndexTest, test_index_and_search) { - Index index(Setup().field(title).field(body)); + Index index(MySetup().field(title).field(body)); index.doc(1) .field(title).add(foo).add(bar).add(foo) .field(body).add(foo).add(foo).add(foo) @@ -254,9 +274,9 @@ TEST("testIndexAndSearch") // tests index update behavior; remove/update and unordered docid // indexing. -TEST("require that documents can be removed and updated") +TEST(MemoryIndexTest, require_that_documents_can_be_removed_and_updated) { - Index index(Setup().field(title)); + Index index(MySetup().field(title)); // add unordered index.doc(3).field(title).add(foo).add(foo).add(foo).commit(); @@ -288,9 +308,9 @@ TEST("require that documents can be removed and updated") // test the fake field source here, to make sure it acts similar to // the memory index field source. -TEST("testFakeSearchable") +TEST(MemoryIndexTest, test_fake_searchable) { - Index index(Setup().field(title).field(body)); + Index index(MySetup().field(title).field(body)); // setup fake field source with predefined results FakeSearchable fakeSource; @@ -340,9 +360,9 @@ TEST("testFakeSearchable") fakeSource, "bogus", makeTerm(foo))); } -TEST("requireThatFrozenIndexIgnoresUpdates") +TEST(MemoryIndexTest, require_that_frozen_index_ignores_updates) { - Index index(Setup().field(title)); + Index index(MySetup().field(title)); Document::UP doc1 = index.doc(1).field(title).add(foo).add(bar).commit(); FakeResult ffr = FakeResult().doc(1).len(2).pos(0); EXPECT_TRUE(verifyResult(ffr, index.index, title, makeTerm(foo))); @@ -355,64 +375,64 @@ TEST("requireThatFrozenIndexIgnoresUpdates") EXPECT_TRUE(verifyResult(ffr, index.index, title, makeTerm(foo))); } -TEST("requireThatNumDocsAndDocIdLimitIsReturned") +TEST(MemoryIndexTest, require_that_num_docs_and_doc_id_limit_is_returned) { - Index index(Setup().field(title)); - EXPECT_EQUAL(0u, index.index.getNumDocs()); - EXPECT_EQUAL(1u, index.index.getDocIdLimit()); + Index index(MySetup().field(title)); + EXPECT_EQ(0u, index.index.getNumDocs()); + EXPECT_EQ(1u, index.index.getDocIdLimit()); Document::UP doc1 = index.doc(1).field(title).add(foo).commit(); - EXPECT_EQUAL(1u, index.index.getNumDocs()); - EXPECT_EQUAL(2u, index.index.getDocIdLimit()); + EXPECT_EQ(1u, index.index.getNumDocs()); + EXPECT_EQ(2u, index.index.getDocIdLimit()); Document::UP doc4 = index.doc(4).field(title).add(foo).commit(); - EXPECT_EQUAL(2u, index.index.getNumDocs()); - EXPECT_EQUAL(5u, index.index.getDocIdLimit()); + EXPECT_EQ(2u, index.index.getNumDocs()); + EXPECT_EQ(5u, index.index.getDocIdLimit()); Document::UP doc2 = index.doc(2).field(title).add(foo).commit(); - EXPECT_EQUAL(3u, index.index.getNumDocs()); - EXPECT_EQUAL(5u, index.index.getDocIdLimit()); + EXPECT_EQ(3u, index.index.getNumDocs()); + EXPECT_EQ(5u, index.index.getDocIdLimit()); // re-add doc4 index.doc(4).field(title).add(bar).commit(); - EXPECT_EQUAL(3u, index.index.getNumDocs()); - EXPECT_EQUAL(5u, index.index.getDocIdLimit()); + EXPECT_EQ(3u, index.index.getNumDocs()); + EXPECT_EQ(5u, index.index.getDocIdLimit()); // remove doc2 index.remove(2); - EXPECT_EQUAL(2u, index.index.getNumDocs()); - EXPECT_EQUAL(5u, index.index.getDocIdLimit()); + EXPECT_EQ(2u, index.index.getNumDocs()); + EXPECT_EQ(5u, index.index.getDocIdLimit()); } -TEST("requireThatWeUnderstandTheMemoryFootprint") +TEST(MemoryIndexTest, require_that_we_understand_the_memory_footprint) { constexpr size_t BASE_SIZE = 188172u; { - Setup setup; + MySetup setup; Index index(setup); - EXPECT_EQUAL(0u, index.index.getStaticMemoryFootprint()); - EXPECT_EQUAL(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().allocatedBytes()); + EXPECT_EQ(0u, index.index.getStaticMemoryFootprint()); + EXPECT_EQ(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().allocatedBytes()); } { - Index index(Setup().field("f1")); - EXPECT_EQUAL(BASE_SIZE, index.index.getStaticMemoryFootprint()); - EXPECT_EQUAL(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().allocatedBytes()); + Index index(MySetup().field("f1")); + EXPECT_EQ(BASE_SIZE, index.index.getStaticMemoryFootprint()); + EXPECT_EQ(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().allocatedBytes()); } { - Index index(Setup().field("f1").field("f2")); - EXPECT_EQUAL(2 * BASE_SIZE, index.index.getStaticMemoryFootprint()); - EXPECT_EQUAL(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().allocatedBytes()); + Index index(MySetup().field("f1").field("f2")); + EXPECT_EQ(2 * BASE_SIZE, index.index.getStaticMemoryFootprint()); + EXPECT_EQ(index.index.getStaticMemoryFootprint(), index.index.getMemoryUsage().allocatedBytes()); } } -TEST("requireThatNumWordsIsReturned") +TEST(MemoryIndexTest, require_that_num_words_is_returned) { - Index index(Setup().field(title)); - EXPECT_EQUAL(0u, index.index.getNumWords()); + Index index(MySetup().field(title)); + EXPECT_EQ(0u, index.index.getNumWords()); index.doc(1).field(title).add(foo).commit(); - EXPECT_EQUAL(1u, index.index.getNumWords()); + EXPECT_EQ(1u, index.index.getNumWords()); index.doc(2).field(title).add(foo).add(bar).add(body).commit(); - EXPECT_EQUAL(3u, index.index.getNumWords()); + EXPECT_EQ(3u, index.index.getNumWords()); } -TEST("requireThatWeCanFakeBitVector") +TEST(MemoryIndexTest, require_that_we_can_fake_bit_vector) { - Index index(Setup().field(title)); + Index index(MySetup().field(title)); index.doc(1).field(title).add(foo).commit(); index.doc(3).field(title).add(foo).commit(); { @@ -437,8 +457,24 @@ TEST("requireThatWeCanFakeBitVector") EXPECT_TRUE(search.get() != NULL); EXPECT_TRUE(dynamic_cast<BooleanMatchIteratorWrapper *>(search.get()) != NULL); search->initFullRange(); - EXPECT_EQUAL("1,3", toString(*search)); + EXPECT_EQ("1,3", toString(*search)); } } -TEST_MAIN() { TEST_RUN_ALL(); } +TEST(MemoryIndexTest, field_length_info_can_be_retrieved_per_field) +{ + Index index(MySetup().field(title).field(body) + .field_length("title", FieldLengthInfo(3, 5)) + .field_length("body", FieldLengthInfo(7, 11))); + + EXPECT_EQ(3, index.index.get_field_length_info("title").get_average_field_length()); + EXPECT_EQ(5, index.index.get_field_length_info("title").get_num_samples()); + + EXPECT_EQ(7, index.index.get_field_length_info("body").get_average_field_length()); + EXPECT_EQ(11, index.index.get_field_length_info("body").get_num_samples()); + + EXPECT_EQ(0, index.index.get_field_length_info("na").get_average_field_length()); + EXPECT_EQ(0, index.index.get_field_length_info("na").get_num_samples()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/index/field_length_calculator.h b/searchlib/src/vespa/searchlib/index/field_length_calculator.h index acbbe38321a..4d4858e38bc 100644 --- a/searchlib/src/vespa/searchlib/index/field_length_calculator.h +++ b/searchlib/src/vespa/searchlib/index/field_length_calculator.h @@ -2,8 +2,10 @@ #pragma once +#include "field_length_info.h" #include <atomic> #include <algorithm> +#include <cstdint> namespace search::index { @@ -29,16 +31,28 @@ public: { } + FieldLengthCalculator(const FieldLengthInfo& info, uint32_t max_num_samples = 100000) + : _average_field_length(info.get_average_field_length()), + _num_samples(std::min(info.get_num_samples(), max_num_samples)), + _max_num_samples(max_num_samples) + { + } + double get_average_field_length() const { return _average_field_length.load(std::memory_order_relaxed); } uint32_t get_num_samples() const { return _num_samples; } - uint32_t get_max_num_samples() { return _max_num_samples; } - + uint32_t get_max_num_samples() const { return _max_num_samples; } + + FieldLengthInfo get_info() const { + return FieldLengthInfo(get_average_field_length(), get_num_samples()); + } + void add_field_length(uint32_t field_length) { if (_num_samples < _max_num_samples) { ++_num_samples; } _average_field_length.store((_average_field_length.load(std::memory_order_relaxed) * (_num_samples - 1) + field_length) / _num_samples, std::memory_order_relaxed); } + }; } diff --git a/searchlib/src/vespa/searchlib/index/field_length_info.h b/searchlib/src/vespa/searchlib/index/field_length_info.h index 3ae3d38d86e..929e12beba9 100644 --- a/searchlib/src/vespa/searchlib/index/field_length_info.h +++ b/searchlib/src/vespa/searchlib/index/field_length_info.h @@ -2,6 +2,8 @@ #pragma once +#include <cstdint> + namespace search::index { /** diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 2243df41b0b..e2e1c99a9b9 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -43,7 +43,12 @@ operator<<(vespalib::asciistream & os, const FieldIndex::WordKey & rhs) return os; } -FieldIndex::FieldIndex(const Schema & schema, uint32_t fieldId) +FieldIndex::FieldIndex(const index::Schema& schema, uint32_t fieldId) + : FieldIndex(schema, fieldId, index::FieldLengthInfo()) +{ +} + +FieldIndex::FieldIndex(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info) : _wordStore(), _numUniqueWords(0), _generationHandler(), @@ -53,8 +58,9 @@ FieldIndex::FieldIndex(const Schema & schema, uint32_t fieldId) _fieldId(fieldId), _remover(_wordStore), _inserter(std::make_unique<OrderedFieldIndexInserter>(*this)), - _calculator() -{ } + _calculator(info) +{ +} FieldIndex::~FieldIndex() { diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h index 85685a5e1d1..dba57f553b5 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h @@ -104,7 +104,8 @@ public: return _featureStore.addFeatures(_fieldId, features).first; } - FieldIndex(const index::Schema &schema, uint32_t fieldId); + FieldIndex(const index::Schema& schema, uint32_t fieldId); + FieldIndex(const index::Schema& schema, uint32_t fieldId, const index::FieldLengthInfo& info); ~FieldIndex(); PostingList::Iterator find(const vespalib::stringref word) const; diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp index b75ea7f0a70..40b1e8f360f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp @@ -4,16 +4,16 @@ #include "field_inverter.h" #include "ordered_field_index_inserter.h" #include <vespa/searchlib/bitcompression/posocccompression.h> - +#include <vespa/searchlib/index/i_field_length_inspector.h> +#include <vespa/vespalib/btree/btree.hpp> +#include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreenodestore.hpp> -#include <vespa/vespalib/btree/btreestore.hpp> -#include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/vespalib/btree/btree.hpp> -#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.memoryindex.field_index_collection"); @@ -22,17 +22,19 @@ LOG_SETUP(".searchlib.memoryindex.field_index_collection"); namespace search { using index::DocIdAndFeatures; -using index::WordDocElementFeatures; +using index::IFieldLengthInspector; using index::Schema; +using index::WordDocElementFeatures; namespace memoryindex { -FieldIndexCollection::FieldIndexCollection(const Schema & schema) +FieldIndexCollection::FieldIndexCollection(const Schema& schema, const IFieldLengthInspector& inspector) : _fieldIndexes(), _numFields(schema.getNumIndexFields()) { for (uint32_t fieldId = 0; fieldId < _numFields; ++fieldId) { - auto fieldIndex = std::make_unique<FieldIndex>(schema, fieldId); + const auto& field = schema.getIndexField(fieldId); + auto fieldIndex = std::make_unique<FieldIndex>(schema, fieldId, inspector.get_field_length_info(field.getName())); _fieldIndexes.push_back(std::move(fieldIndex)); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h index d5212c41819..53f42658d0a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h @@ -5,6 +5,8 @@ #include "i_field_index_collection.h" #include "field_index.h" +namespace search::index { class IFieldLengthInspector; } + namespace search::memoryindex { class IFieldIndexRemoveListener; @@ -27,7 +29,7 @@ private: uint32_t _numFields; public: - FieldIndexCollection(const index::Schema &schema); + FieldIndexCollection(const index::Schema& schema, const index::IFieldLengthInspector& inspector); ~FieldIndexCollection(); PostingList::Iterator find(const vespalib::stringref word, uint32_t fieldId) const { diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index e2cac316580..6686745f8c2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -25,6 +25,8 @@ using vespalib::GenerationHandler; namespace search { using fef::TermFieldMatchDataArray; +using index::FieldLengthInfo; +using index::IFieldLengthInspector; using index::IndexBuilder; using index::Schema; using index::SchemaUtil; @@ -53,13 +55,14 @@ using queryeval::IRequestContext; namespace search::memoryindex { -MemoryIndex::MemoryIndex(const Schema &schema, - ISequencedTaskExecutor &invertThreads, - ISequencedTaskExecutor &pushThreads) +MemoryIndex::MemoryIndex(const Schema& schema, + const IFieldLengthInspector& inspector, + ISequencedTaskExecutor& invertThreads, + ISequencedTaskExecutor& pushThreads) : _schema(schema), _invertThreads(invertThreads), _pushThreads(pushThreads), - _fieldIndexes(std::make_unique<FieldIndexCollection>(_schema)), + _fieldIndexes(std::make_unique<FieldIndexCollection>(_schema, inspector)), _inverter0(std::make_unique<DocumentInverter>(_schema, _invertThreads, _pushThreads, *_fieldIndexes)), _inverter1(std::make_unique<DocumentInverter>(_schema, _invertThreads, _pushThreads, *_fieldIndexes)), _inverter(_inverter0.get()), @@ -290,4 +293,14 @@ MemoryIndex::getPrunedSchema() const return _prunedSchema; } +FieldLengthInfo +MemoryIndex::get_field_length_info(const vespalib::string& field_name) const +{ + uint32_t field_id = _schema.getIndexFieldId(field_name); + if (field_id != Schema::UNKNOWN_FIELD_ID) { + return _fieldIndexes->get_calculator(field_id).get_info(); + } + return FieldLengthInfo(); +} + } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index c350da31c54..44252aa2cdc 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -2,13 +2,17 @@ #pragma once +#include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/searchlib/index/field_length_info.h> #include <vespa/searchlib/queryeval/searchable.h> -#include <vespa/searchcommon/common/schema.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/memoryusage.h> -namespace search::index { class IndexBuilder; } +namespace search::index { + class IFieldLengthInspector; + class IndexBuilder; +} namespace search { class ISequencedTaskExecutor; } @@ -82,13 +86,15 @@ public: * Create a new memory index based on the given schema. * * @param schema the schema with which text and uri fields to keep in the index. + * @param inspector the inspector used to lookup initial field length info for all index fields. * @param invertThreads the executor with threads for doing document inverting. * @param pushThreads the executor with threads for doing pushing of changes (inverted documents) * to corresponding field indexes. */ - MemoryIndex(const index::Schema &schema, - ISequencedTaskExecutor &invertThreads, - ISequencedTaskExecutor &pushThreads); + MemoryIndex(const index::Schema& schema, + const index::IFieldLengthInspector& inspector, + ISequencedTaskExecutor& invertThreads, + ISequencedTaskExecutor& pushThreads); ~MemoryIndex(); @@ -165,6 +171,8 @@ public: vespalib::MemoryUsage getMemoryUsage() const; uint64_t getStaticMemoryFootprint() const { return _staticMemoryFootprint; } + + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const; }; } diff --git a/searchlib/src/vespa/searchlib/test/index/mock_field_length_inspector.h b/searchlib/src/vespa/searchlib/test/index/mock_field_length_inspector.h new file mode 100644 index 00000000000..4954e732862 --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/index/mock_field_length_inspector.h @@ -0,0 +1,19 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/index/i_field_length_inspector.h> + +namespace search::index::test { + +/** + * Mock of IFieldLengthInspector returning empty field info for all fields. + */ +class MockFieldLengthInspector : public IFieldLengthInspector { + FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { + (void) field_name; + return FieldLengthInfo(); + } +}; + +} diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index cdaa6e9aaa3..2ddf41236c9 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -544,6 +544,10 @@ public: } uint32_t populate_bucket_db_via_request_bucket_info_for_benchmarking(); + + void complete_recovery_mode() { + _distributor->scanAllBuckets(); + } }; BucketDBUpdaterTest::BucketDBUpdaterTest() @@ -1900,8 +1904,8 @@ TEST_F(BucketDBUpdaterTest, testClusterStateAlwaysSendsFullFetchWhenDistribution TEST_F(BucketDBUpdaterTest, testChangedDistributionConfigTriggersRecoveryMode) { ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), messageCount(6), 20)); _sender.clear(); - // First cluster state; implicit scan of all buckets which does not - // use normal recovery mode ticking-path. + EXPECT_TRUE(_distributor->isInRecoveryMode()); + complete_recovery_mode(); EXPECT_FALSE(_distributor->isInRecoveryMode()); std::string distConfig(getDistConfig6Nodes4Groups()); @@ -1920,6 +1924,8 @@ TEST_F(BucketDBUpdaterTest, testChangedDistributionConfigTriggersRecoveryMode) { // Pending cluster state (i.e. distribution) has been enabled, which should // cause recovery mode to be entered. EXPECT_TRUE(_distributor->isInRecoveryMode()); + complete_recovery_mode(); + EXPECT_FALSE(_distributor->isInRecoveryMode()); } namespace { diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 46c756001d9..c519ef0713b 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -1072,6 +1072,7 @@ void Distributor_Test::leaving_recovery_mode_immediately_sends_getnodestate_repl void Distributor_Test::do_test_pending_merge_getnodestate_reply_edge(BucketSpace space) { setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2"); + CPPUNIT_ASSERT(_distributor->isInRecoveryMode()); // 2 buckets with missing replicas triggering merge pending stats addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a"); addNodesToBucketDB(Bucket(space, BucketId(16, 2)), "0=1/1/1/t/a"); @@ -1079,29 +1080,30 @@ void Distributor_Test::do_test_pending_merge_getnodestate_reply_edge(BucketSpace CPPUNIT_ASSERT(!_distributor->isInRecoveryMode()); const auto space_name = FixedBucketSpaces::to_string(space); assertBucketSpaceStats(2, 0, 1, space_name, _distributor->getBucketSpacesStats()); - CPPUNIT_ASSERT_EQUAL(size_t(0), explicit_node_state_reply_send_invocations()); + // First completed scan sends off merge stats et al to cluster controller + CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations()); // Edge not triggered when 1 bucket with missing replica left addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a,1=1/1/1/t"); tickDistributorNTimes(3); assertBucketSpaceStats(1, 1, 1, space_name, _distributor->getBucketSpacesStats()); - CPPUNIT_ASSERT_EQUAL(size_t(0), explicit_node_state_reply_send_invocations()); + CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations()); // Edge triggered when no more buckets with requiring merge addNodesToBucketDB(Bucket(space, BucketId(16, 2)), "0=1/1/1/t/a,1=1/1/1/t"); tickDistributorNTimes(3); assertBucketSpaceStats(0, 2, 1, space_name, _distributor->getBucketSpacesStats()); - CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations()); + CPPUNIT_ASSERT_EQUAL(size_t(2), explicit_node_state_reply_send_invocations()); // Should only send when edge happens, not in subsequent DB iterations tickDistributorNTimes(10); - CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations()); + CPPUNIT_ASSERT_EQUAL(size_t(2), explicit_node_state_reply_send_invocations()); // Going back to merges pending should _not_ send a getnodestate reply (at least for now) addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a"); tickDistributorNTimes(3); assertBucketSpaceStats(1, 1, 1, space_name, _distributor->getBucketSpacesStats()); - CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations()); + CPPUNIT_ASSERT_EQUAL(size_t(2), explicit_node_state_reply_send_invocations()); } void Distributor_Test::pending_to_no_pending_default_merges_edge_immediately_sends_getnodestate_replies() { diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 771baa04fca..7cb7d687446 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -359,12 +359,10 @@ Distributor::enableClusterStateBundle(const lib::ClusterStateBundle& state) if (!_doneInitializing && baselineState.getNodeState(myNode).getState() == lib::State::UP) { - scanAllBuckets(); _doneInitializing = true; _doneInitializeHandler.notifyDoneInitializing(); - } else { - enterRecoveryMode(); } + enterRecoveryMode(); // Clear all active messages on nodes that are down. for (uint16_t i = 0; i < baselineState.getNodeCount(lib::NodeType::STORAGE); ++i) { |