diff options
36 files changed, 428 insertions, 339 deletions
diff --git a/config-lib/src/main/java/com/yahoo/config/BooleanNode.java b/config-lib/src/main/java/com/yahoo/config/BooleanNode.java index d7ca5b55b55..cb91912bd5a 100644 --- a/config-lib/src/main/java/com/yahoo/config/BooleanNode.java +++ b/config-lib/src/main/java/com/yahoo/config/BooleanNode.java @@ -5,6 +5,7 @@ package com.yahoo.config; * The BooleanNode class represents a boolean in a configuration tree. */ public class BooleanNode extends LeafNode<Boolean> { + public BooleanNode() { } diff --git a/config-lib/src/main/java/com/yahoo/config/ChangesRequiringRestart.java b/config-lib/src/main/java/com/yahoo/config/ChangesRequiringRestart.java index 3eeb637a991..3cd7c5541a9 100644 --- a/config-lib/src/main/java/com/yahoo/config/ChangesRequiringRestart.java +++ b/config-lib/src/main/java/com/yahoo/config/ChangesRequiringRestart.java @@ -63,8 +63,8 @@ public class ChangesRequiringRestart { } } - private ArrayList<ReportLine> report = new ArrayList<>(); - private String componentName; + private final ArrayList<ReportLine> report = new ArrayList<>(); + private final String componentName; public ChangesRequiringRestart(String componentName) { this.componentName = componentName; @@ -105,14 +105,14 @@ public class ChangesRequiringRestart { int commonElements = Math.min(from.size(), to.size()); for (int i = 0; i < commonElements; ++i) { ChangesRequiringRestart childReport = func.getChangesRequiringRestart(from.get(i), to.get(i)); - String prefix = childReport.componentName + "[" + Integer.toString(i) + "]"; + String prefix = childReport.componentName + "[" + i + "]"; mergeChanges(prefix, childReport); } for (int i = commonElements; i < from.size(); ++i) { - report.add(new ReportLine(name + "[" + Integer.toString(i) + "]", from.get(i), null, comment)); + report.add(new ReportLine(name + "[" + i + "]", from.get(i), null, comment)); } for (int i = commonElements; i < to.size(); ++i) { - report.add(new ReportLine(name + "[" + Integer.toString(i) + "]", null, to.get(i), comment)); + report.add(new ReportLine(name + "[" + i + "]", null, to.get(i), comment)); } } return this; diff --git a/config-lib/src/main/java/com/yahoo/config/DoubleNode.java b/config-lib/src/main/java/com/yahoo/config/DoubleNode.java index 5be0c2dc4c8..c90284f9e2e 100644 --- a/config-lib/src/main/java/com/yahoo/config/DoubleNode.java +++ b/config-lib/src/main/java/com/yahoo/config/DoubleNode.java @@ -5,6 +5,7 @@ package com.yahoo.config; * The DoubleNode class represents a double in a configuration tree. */ public class DoubleNode extends LeafNode<Double> { + public DoubleNode() { } diff --git a/config-lib/src/main/java/com/yahoo/config/EnumNode.java b/config-lib/src/main/java/com/yahoo/config/EnumNode.java index b3969dcfd8c..5427e6dc057 100644 --- a/config-lib/src/main/java/com/yahoo/config/EnumNode.java +++ b/config-lib/src/main/java/com/yahoo/config/EnumNode.java @@ -5,6 +5,7 @@ package com.yahoo.config; * The EnumNode class is a superclass for Enumerations in a configuration tree. */ public abstract class EnumNode<ENUM extends Enum<?>> extends LeafNode<ENUM> { + public EnumNode() { } diff --git a/config-lib/src/main/java/com/yahoo/config/FileNode.java b/config-lib/src/main/java/com/yahoo/config/FileNode.java index a7c1ebb1488..170c1e438a9 100644 --- a/config-lib/src/main/java/com/yahoo/config/FileNode.java +++ b/config-lib/src/main/java/com/yahoo/config/FileNode.java @@ -36,5 +36,4 @@ public class FileNode extends LeafNode<FileReference> { return true; } - } diff --git a/config-lib/src/main/java/com/yahoo/config/InnerNode.java b/config-lib/src/main/java/com/yahoo/config/InnerNode.java index 610221094d2..ac60ee9a6e9 100644 --- a/config-lib/src/main/java/com/yahoo/config/InnerNode.java +++ b/config-lib/src/main/java/com/yahoo/config/InnerNode.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Iterator; import java.util.Map; +import java.util.Objects; /** * Superclass for all inner nodes in a {@link ConfigInstance}. @@ -44,7 +45,7 @@ public abstract class InnerNode extends Node { /** * Overrides {@link Node#postInitialize(String)}. - * Perform post initialization on this nodes children. + * Perform post initialization on this node's children. * * @param configId The config id of this instance. */ @@ -65,7 +66,7 @@ public abstract class InnerNode extends Node { return false; /* This implementation requires getChildren() to return elements in order. - Hence we should make it final. Or make equals independent of order. */ + Hence, we should make it final. Or make equals independent of order. */ Collection<Object> children = getChildren().values(); Collection<Object> otherChildren = ((InnerNode)other).getChildren().values(); @@ -74,7 +75,7 @@ public abstract class InnerNode extends Node { while(e1.hasNext() && e2.hasNext()) { Object o1 = e1.next(); Object o2 = e2.next(); - if (!(o1 == null ? o2 == null : o1.equals(o2))) + if (!(Objects.equals(o1, o2))) return false; } return !(e1.hasNext() || e2.hasNext()); @@ -135,7 +136,6 @@ public abstract class InnerNode extends Node { ret.put(name + "{" + entry.getKey() + "}", entry.getValue()); } - private static void addNodeVectorEntries(Map<String, Node> ret, String name, NodeVector<?> vector) { for (int j = 0; j < vector.length(); j++) ret.put(name + "[" + j + "]", (Node) vector.get(j)); diff --git a/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java b/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java index 8bf9b4c1520..30a7580c6dd 100644 --- a/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java @@ -5,7 +5,6 @@ import java.util.List; /** * @author gjoranv - * @since 5.1.4 */ public class InnerNodeVector<NODE extends InnerNode> extends NodeVector<NODE> { diff --git a/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java b/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java index ddbed4258dc..a4fea95088d 100644 --- a/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java @@ -57,7 +57,7 @@ public class LeafNodeVector<REAL, NODE extends LeafNode<REAL>> extends NodeVecto // TODO: Try to eliminate the need for this method when we have moved FileAcquirer to the config library // It is needed now because the builder has a list of String, while REAL=FileReference. public static LeafNodeVector<FileReference, FileNode> createFileNodeVector(Collection<String> values) { - List<FileReference> fileReferences = new ArrayList<FileReference>(); + List<FileReference> fileReferences = new ArrayList<>(); for (String s : values) fileReferences.add(new FileReference(ReferenceNode.stripQuotes(s))); diff --git a/config-lib/src/main/java/com/yahoo/config/LongNode.java b/config-lib/src/main/java/com/yahoo/config/LongNode.java index 02b6a5f7654..ac6ae457087 100755 --- a/config-lib/src/main/java/com/yahoo/config/LongNode.java +++ b/config-lib/src/main/java/com/yahoo/config/LongNode.java @@ -48,4 +48,5 @@ public class LongNode extends LeafNode<Long> { void serialize(Serializer serializer) { serializer.serialize(value); } + } diff --git a/config-lib/src/main/java/com/yahoo/config/ModelReference.java b/config-lib/src/main/java/com/yahoo/config/ModelReference.java index fbf34c7ff23..38ddf753bd6 100644 --- a/config-lib/src/main/java/com/yahoo/config/ModelReference.java +++ b/config-lib/src/main/java/com/yahoo/config/ModelReference.java @@ -67,8 +67,8 @@ public class ModelReference { public String toString() { if (resolved != null) return resolved.toString(); return modelId.orElse("\"\"") + " " + - url.map(v -> v.value()).orElse("\"\"") + " " + - path.map(v -> v.value()).orElse("\"\""); + url.map(UrlReference::value).orElse("\"\"") + " " + + path.map(FileReference::value).orElse("\"\""); } /** diff --git a/config-lib/src/main/java/com/yahoo/config/Node.java b/config-lib/src/main/java/com/yahoo/config/Node.java index a07409b19e7..188a9ecc623 100644 --- a/config-lib/src/main/java/com/yahoo/config/Node.java +++ b/config-lib/src/main/java/com/yahoo/config/Node.java @@ -9,16 +9,16 @@ package com.yahoo.config; public abstract class Node { /** - * Postinitialize this node. Any node needing to process its values depending on the config + * Post-initialize this node. Any node needing to process its values depending on the config * id should override this method. * * @param configId the configId of the ConfigInstance that owns (or is) this node */ - public void postInitialize(String configId) { return; } + public void postInitialize(String configId) {} /** * This method is meant for internal use in the configuration system. - * Overrides Object.clone(), and is overriden by LeafNode.clone(). + * Overrides Object.clone(), and is overridden by LeafNode.clone(). * * @return a new instance similar to this object. */ diff --git a/config-lib/src/main/java/com/yahoo/config/NodeVector.java b/config-lib/src/main/java/com/yahoo/config/NodeVector.java index 3d07de944fd..5ab1f97ecde 100644 --- a/config-lib/src/main/java/com/yahoo/config/NodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/NodeVector.java @@ -12,7 +12,7 @@ import java.util.*; */ public abstract class NodeVector<NODE> implements java.util.List<NODE> { - protected final ArrayList<NODE> vector = new ArrayList<NODE>(); + protected final ArrayList<NODE> vector = new ArrayList<>(); /** * Returns the number of elements in this NodeVector. @@ -24,7 +24,6 @@ public abstract class NodeVector<NODE> implements java.util.List<NODE> { return size(); } - @SuppressWarnings("serial") public static class ReadOnlyException extends RuntimeException { } @@ -88,7 +87,6 @@ public abstract class NodeVector<NODE> implements java.util.List<NODE> { return vector.hashCode(); } - @SuppressWarnings("unchecked") public NODE get(int index) { return vector.get(index); } diff --git a/config-lib/src/main/java/com/yahoo/config/Serializer.java b/config-lib/src/main/java/com/yahoo/config/Serializer.java index 12c45c789a4..5e2d35cb2af 100644 --- a/config-lib/src/main/java/com/yahoo/config/Serializer.java +++ b/config-lib/src/main/java/com/yahoo/config/Serializer.java @@ -3,7 +3,6 @@ package com.yahoo.config; /** * @author Ulf Lilleengen -* @since 5.1 */ public interface Serializer { Serializer createInner(String name); @@ -28,4 +27,5 @@ public interface Serializer { void serialize(long value); void serialize(int value); void serialize(String value); + } diff --git a/config-lib/src/main/java/com/yahoo/config/StringNode.java b/config-lib/src/main/java/com/yahoo/config/StringNode.java index 2e4f0df07af..0a87a09ddc6 100644 --- a/config-lib/src/main/java/com/yahoo/config/StringNode.java +++ b/config-lib/src/main/java/com/yahoo/config/StringNode.java @@ -29,10 +29,10 @@ public class StringNode extends LeafNode<String> { /** * Returns the value of this string. Same as {@link #getValue()} * since the value of this node is a String (but implementations - * in other {@link LeafNode} sub-classes differ). + * in other {@link LeafNode} subclasses differ). * * @return the string representation of this StringNode, or null if - * the vlaue is explicitly set to null + * the value is explicitly set to null */ public String value() { return value; @@ -63,30 +63,18 @@ public class StringNode extends LeafNode<String> { throw new IllegalArgumentException("Parse error" + string); } switch (sb.charAt(i)) { - case'n': - sb.setCharAt(i, '\n'); - break; - case'r': - sb.setCharAt(i, '\r'); - break; - case't': - sb.setCharAt(i, '\t'); - break; - case'f': - sb.setCharAt(i, '\f'); - break; - case'x': + case 'n' -> sb.setCharAt(i, '\n'); + case 'r' -> sb.setCharAt(i, '\r'); + case 't' -> sb.setCharAt(i, '\t'); + case 'f' -> sb.setCharAt(i, '\f'); + case 'x' -> { if (i + 2 >= sb.length()) { - throw new IllegalArgumentException - ("Could not parse hex value " + string); + throw new IllegalArgumentException("Could not parse hex value " + string); } - sb.setCharAt(i, (char) Integer.parseInt - (sb.substring(i + 1, i + 3), 16)); + sb.setCharAt(i, (char) Integer.parseInt(sb.substring(i + 1, i + 3), 16)); sb.delete(i + 1, i + 3); - break; - case'\\': - sb.setCharAt(i, '\\'); - break; + } + case '\\' -> sb.setCharAt(i, '\\'); } } } @@ -101,7 +89,7 @@ public class StringNode extends LeafNode<String> { } /** - * Sets the value of this string from a the string representation + * Sets the value of this string from the string representation * of this value in the (escaped) input configuration. The value * supplied to this method needs un-escaping and will be * un-escaped. diff --git a/config-lib/src/main/java/com/yahoo/config/text/StringUtilities.java b/config-lib/src/main/java/com/yahoo/config/text/StringUtilities.java index ca0bc1758e3..5a1a570e913 100644 --- a/config-lib/src/main/java/com/yahoo/config/text/StringUtilities.java +++ b/config-lib/src/main/java/com/yahoo/config/text/StringUtilities.java @@ -20,9 +20,9 @@ public class StringUtilities { private static class ReplacementCharacters { - public byte[] needEscape = new byte[256]; - public byte[] replacement1 = new byte[256]; - public byte[] replacement2 = new byte[256]; + public final byte[] needEscape = new byte[256]; + public final byte[] replacement1 = new byte[256]; + public final byte[] replacement2 = new byte[256]; public ReplacementCharacters() { for (int i=0; i<256; ++i) { diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java index 76871aaca42..15e35682980 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java @@ -15,7 +15,6 @@ import java.lang.reflect.Method; import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import static com.yahoo.foo.StructtypesConfig.Simple.Gender.Enum.FEMALE; import static com.yahoo.test.FunctionTestConfig.BasicStruct; @@ -257,9 +256,8 @@ public class ConfigInstanceBuilderTest { assertEquals(configId, config.refwithdef()); assertEquals("etc", config.fileVal().value()); assertEquals(1, config.boolarr().size()); - assertEquals(1, config.boolarr().size()); // new api with accessor for a List of the original Java type - assertFalse(config.boolarr().get(0)); // new List api - assertFalse(config.boolarr(0)); // short-hand + assertFalse(config.boolarr().get(0)); + assertFalse(config.boolarr(0)); // shorthand assertEquals(0, config.intarr().size()); assertEquals(2, config.longarr().size()); assertEquals(Long.MAX_VALUE, config.longarr(0)); @@ -267,7 +265,7 @@ public class ConfigInstanceBuilderTest { assertEquals(2, config.doublearr().size()); assertEquals(1, config.stringarr().size()); assertEquals(1, config.enumarr().size()); - assertEquals(FunctionTestConfig.Enumarr.VALUES, config.enumarr().get(0)); // new List api, don't have to call value() + assertEquals(FunctionTestConfig.Enumarr.VALUES, config.enumarr().get(0)); assertEquals(3, config.refarr().size()); assertEquals(1, config.fileArr().size()); assertEquals(configId, config.refarr(0)); @@ -283,10 +281,10 @@ public class ConfigInstanceBuilderTest { assertEquals("basicFoo", config.basicStruct().foo()); assertEquals(3, config.basicStruct().bar()); // new List api assertEquals(2, config.basicStruct().intArr().size()); - assertEquals(310, config.basicStruct().intArr().get(0).intValue()); // new List api - assertEquals(311, config.basicStruct().intArr().get(1).intValue()); // new List api - assertEquals(310, config.basicStruct().intArr(0)); // short-hand - assertEquals("inner0", config.rootStruct().inner0().name()); // new List api + assertEquals(310, config.basicStruct().intArr().get(0).intValue()); + assertEquals(311, config.basicStruct().intArr().get(1).intValue()); + assertEquals(310, config.basicStruct().intArr(0)); // shorthand + assertEquals("inner0", config.rootStruct().inner0().name()); assertEquals(11, config.rootStruct().inner0().index()); assertEquals("inner1", config.rootStruct().inner1().name()); assertEquals(12, config.rootStruct().inner1().index()); @@ -297,8 +295,8 @@ public class ConfigInstanceBuilderTest { assertEquals("blue a=\"escaped\"", config.rootStruct().innerArr(1).stringVal()); assertEquals(2, config.myarray().size()); // new List api - assertEquals(configId, config.myarray().get(0).refval()); // new List api - assertEquals(configId, config.myarray(0).refval()); // short-hand + assertEquals(configId, config.myarray().get(0).refval()); + assertEquals(configId, config.myarray(0).refval()); // shorthand assertEquals("file0", config.myarray(0).fileVal().value()); assertEquals(1, config.myarray(0).myStruct().a()); assertEquals(2, config.myarray(0).myStruct().b()); diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java index db1509fba93..4a16eebfeb9 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config; -import com.yahoo.test.AppConfig; import com.yahoo.test.FunctionTestConfig; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,11 +47,6 @@ public class ConfigInstanceEqualsTest { } @Test - void require_false_for_different_subclass() { - assertNotEquals(config1, new AppConfig(new AppConfig.Builder())); - } - - @Test void require_false_for_different_scalars_at_root_node() { assertNotEquals(config1, new FunctionTestConfig(newBuilder().bool_val(true))); assertNotEquals(config1, new FunctionTestConfig(newBuilder().int_val(0))); diff --git a/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java b/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java index d53db9fa9c7..da82ec69a62 100644 --- a/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java @@ -23,7 +23,7 @@ public class EnumNodeTest { try { value = Enum.valueOf(name); return true; - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException ignored) { } return false; } diff --git a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java index fb751931062..b5080f76605 100644 --- a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java +++ b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java @@ -4,6 +4,7 @@ package com.yahoo.config; import org.junit.jupiter.api.Test; import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -17,72 +18,52 @@ public class NodeVectorTest { @Test void require_that_add_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").add(barNode()); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").add(barNode())); } @Test void require_that_addindex_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").add(0, barNode()); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").add(0, barNode())); } @Test void require_that_addAll_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").addAll(Arrays.asList(barNode())); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").add(barNode())); } @Test void require_that_addAllindex_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").addAll(0, Arrays.asList(barNode())); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").addAll(0, List.of(barNode()))); } @Test void require_that_clear_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").clear(); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").clear()); } @Test void require_that_remove_index_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").remove(0); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").remove(0)); } @Test void require_that_remove_object_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").remove(null); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").remove(null)); } @Test void require_that_removeAll_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").removeAll(null); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").removeAll(null)); } @Test void require_that_retainAll_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").retainAll(null); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").retainAll(null)); } @Test void require_that_set_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").set(0, null); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").set(0, null)); } @Test @@ -91,7 +72,7 @@ public class NodeVectorTest { TestNodeVector v = new TestNodeVector(val.getValue()); assertTrue(v.contains(val)); assertFalse(v.contains(barNode())); - assertTrue(v.containsAll(Arrays.asList(val))); + assertTrue(v.contains(val)); assertFalse(v.containsAll(Arrays.asList(val, barNode()))); } diff --git a/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java b/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java index 3470c96bbcc..7211f8ad776 100644 --- a/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java @@ -23,9 +23,7 @@ public class StringNodeTest { @Test void testUnescapedQuotedStringExceptions() { - assertThrows(IllegalArgumentException.class, () -> { - StringNode.unescapeQuotedString("foo\\"); - }); + assertThrows(IllegalArgumentException.class, () -> StringNode.unescapeQuotedString("foo\\")); } @Test diff --git a/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java b/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java index 39e5f64e5e2..8d15832b73a 100644 --- a/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java +++ b/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java @@ -12,8 +12,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; * @author gjoranv */ public class NamespaceAndPackageTest { - private static String NAMESPACE = "my.namespace"; - private static String PACKAGE = "com.github.myproject"; + private static final String NAMESPACE = "my.namespace"; + private static final String PACKAGE = "com.github.myproject"; @Test void namespace_is_set_from_def_file() { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index 2a382d22a68..e4775fba46f 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -14,7 +14,6 @@ class MetricDefinitions { static final String METHOD_DIMENSION = "httpMethod"; static final String SCHEME_DIMENSION = "scheme"; static final String REQUEST_TYPE_DIMENSION = "requestType"; - static final String CLIENT_IP_DIMENSION = "clientIp"; static final String CLIENT_AUTHENTICATED_DIMENSION = "clientAuthenticated"; static final String REQUEST_SERVER_NAME_DIMENSION = "requestServerName"; static final String FILTER_CHAIN_ID_DIMENSION = "chainId"; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java index d96989830ab..817a99bb57f 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java @@ -7,11 +7,8 @@ import org.eclipse.jetty.io.ssl.SslHandshakeListener; import javax.net.ssl.SSLHandshakeException; import java.util.HashMap; import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.regex.Pattern; /** * A {@link SslHandshakeListener} that reports metrics for SSL handshake failures. @@ -45,8 +42,6 @@ class SslHandshakeFailedListener implements SslHandshakeListener { Map<String, Object> dimensions = new HashMap<>(); dimensions.put(MetricDefinitions.NAME_DIMENSION, connectorName); dimensions.put(MetricDefinitions.PORT_DIMENSION, listenPort); - Optional.ofNullable(event.getSSLEngine().getPeerHost()) - .ifPresent(clientIp -> dimensions.put(MetricDefinitions.CLIENT_IP_DIMENSION, clientIp)); return Map.copyOf(dimensions); } } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListenerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListenerTest.java index 3858d6b9e4e..ad34cc5f024 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListenerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListenerTest.java @@ -25,7 +25,7 @@ public class SslHandshakeFailedListenerTest { @Test void includes_client_ip_dimension_present_when_peer_available() { listener.handshakeFailed(handshakeEvent(true), new SSLHandshakeException("Empty server certificate chain")); - verify(metrics).createContext(eq(Map.of("clientIp", "127.0.0.1", "serverName", "connector", "serverPort", 1234))); + verify(metrics).createContext(eq(Map.of("serverName", "connector", "serverPort", 1234))); } @Test diff --git a/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp b/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp index e3393dc2de7..6c0726dab37 100644 --- a/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp +++ b/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp @@ -134,15 +134,15 @@ Optimize universal_only() { return Optimize::specific("universal_only", my_optimizer); } -using cost_map_t = std::map<vespalib::string,double>; -std::vector<std::pair<vespalib::string,cost_map_t>> benchmark_results; +using cost_list_t = std::vector<std::pair<vespalib::string,double>>; +std::vector<std::pair<vespalib::string,cost_list_t>> benchmark_results; -void benchmark(const vespalib::string &desc, const vespalib::string &expr, std::vector<Optimize> list) { +void benchmark(const vespalib::string &expr, std::vector<Optimize> list) { auto fun = Function::parse(expr); ASSERT_FALSE(fun->has_error()); auto expected = eval_ref(*fun); - cost_map_t cost_map; - fprintf(stderr, "BENCH: %s (%s)\n", desc.c_str(), expr.c_str()); + cost_list_t cost_list; + fprintf(stderr, "BENCH: %s\n", expr.c_str()); for (Optimize &optimize: list) { std::vector<Value::UP> values; for (size_t i = 0; i < fun->num_params(); ++i) { @@ -179,29 +179,27 @@ void benchmark(const vespalib::string &desc, const vespalib::string &expr, std:: ASSERT_NE(optimized, nullptr); CTFMetaData ctf_meta; InterpretedFunction ifun(prod_factory, *optimized, &ctf_meta); + InterpretedFunction::ProfiledContext pctx(ifun); ASSERT_EQ(ctf_meta.steps.size(), ifun.program_size()); - BenchmarkTimer timer(budget); + EXPECT_EQ(spec_from_value(ifun.eval(pctx.context, params)), expected); + EXPECT_EQ(spec_from_value(ifun.eval(pctx, params)), expected); std::vector<duration> prev_time(ctf_meta.steps.size(), duration::zero()); std::vector<duration> min_time(ctf_meta.steps.size(), duration::max()); - InterpretedFunction::ProfiledContext pctx(ifun); - for (bool first = true; timer.has_budget(); first = false) { - const Value &profiled_result = ifun.eval(pctx, params); - if (first) { - EXPECT_EQ(spec_from_value(profiled_result), expected); - } + BenchmarkTimer timer(budget); + while (timer.has_budget()) { timer.before(); const Value &result = ifun.eval(pctx.context, params); + (void) result; timer.after(); - if (first) { - EXPECT_EQ(spec_from_value(result), expected); - } + const Value &profiled_result = ifun.eval(pctx, params); + (void) profiled_result; for (size_t i = 0; i < ctf_meta.steps.size(); ++i) { min_time[i] = std::min(min_time[i], pctx.cost[i].second - prev_time[i]); prev_time[i] = pctx.cost[i].second; } } double cost_us = timer.min_time() * 1000.0 * 1000.0; - cost_map.emplace(optimize.name, cost_us); + cost_list.emplace_back(optimize.name, cost_us); fprintf(stderr, " optimized with: %s: %g us {\n", optimize.name.c_str(), cost_us); for (size_t i = 0; i < ctf_meta.steps.size(); ++i) { auto name = strip_ns(ctf_meta.steps[i].class_name); @@ -213,7 +211,7 @@ void benchmark(const vespalib::string &desc, const vespalib::string &expr, std:: fprintf(stderr, " }\n"); } fprintf(stderr, "\n"); - benchmark_results.emplace_back(desc, std::move(cost_map)); + benchmark_results.emplace_back(expr, std::move(cost_list)); } TensorSpec perform_dot_product(const TensorSpec &a, const TensorSpec &b, const std::vector<vespalib::string> &dims) @@ -266,35 +264,43 @@ TEST(UniversalDotProductTest, bench_vector_dot_product) { } auto optimize_list = std::vector<Optimize>({baseline(), with_universal(), universal_only()}); - benchmark("number number", "reduce(1.0*2.0,sum)", optimize_list); - benchmark("number vector", "reduce(5.0*x128,sum,x)", optimize_list); - benchmark("vector vector small", "reduce(x16*x16,sum,x)", optimize_list); - benchmark("vector vector large", "reduce(x768*x768,sum,x)", optimize_list); - benchmark("vector matrix full", "reduce(y64*x8y64,sum,x,y)", optimize_list); - benchmark("vector matrix inner", "reduce(y64*x8y64,sum,y)", optimize_list); - benchmark("vector matrix outer", "reduce(y64*x8y64,sum,x)", optimize_list); - benchmark("matrix matrix same", "reduce(a8y64*a8y64,sum,y)", optimize_list); - benchmark("matrix matrix different", "reduce(a8y64*b8y64,sum,y)", optimize_list); - benchmark("matmul", "reduce(a8b64*b64c8,sum,b)", optimize_list); - benchmark("sparse overlap", "reduce(x64_1*x64_1,sum,x)", optimize_list); - benchmark("sparse no overlap", "reduce(a64_1*b64_1,sum,b)", optimize_list); - benchmark("mixed dense", "reduce(a1_16x768*x768,sum,x)", optimize_list); - benchmark("mixed mixed complex", "reduce(a1_1x128*a2_1b64_1x128,sum,a,x)", optimize_list); + benchmark("reduce(1.0*2.0,sum)", optimize_list); + benchmark("reduce(5.0*x128,sum,x)", optimize_list); + benchmark("reduce(x16*x16,sum,x)", optimize_list); + benchmark("reduce(x768*x768,sum,x)", optimize_list); + benchmark("reduce(y64*x8y64,sum,x,y)", optimize_list); + benchmark("reduce(y64*x8y64,sum,y)", optimize_list); + benchmark("reduce(y64*x8y64,sum,x)", optimize_list); + benchmark("reduce(a8y64*a8y64,sum,y)", optimize_list); + benchmark("reduce(a8y64*a8y64,sum,a)", optimize_list); + benchmark("reduce(a8y64*b8y64,sum,y)", optimize_list); + benchmark("reduce(a8b64*b64c8,sum,b)", optimize_list); + benchmark("reduce(x64_1*x64_1,sum,x)", optimize_list); + benchmark("reduce(a64_1*b64_1,sum,b)", optimize_list); + benchmark("reduce(a8_1b8_1*b8_1c8_1,sum,b)", optimize_list); + benchmark("reduce(a8_1b8_1*b8_1c8_1,sum,a,c)", optimize_list); + benchmark("reduce(a8_1b8_1*b8_1c8_1,sum,a,b,c)", optimize_list); + benchmark("reduce(b64_1x128*x128,sum,x)", optimize_list); + benchmark("reduce(b64_1x8y128*x8y128,sum,y)", optimize_list); + benchmark("reduce(b64_1x128*x128,sum,b,x)", optimize_list); + benchmark("reduce(a1_1x128*a2_1b64_1x128,sum,a,x)", optimize_list); + benchmark("reduce(x0_0*y8_1,sum,y)", optimize_list); + benchmark("reduce(x8_1*y0_0,sum,y)", optimize_list); - size_t max_desc_size = 0; - for (const auto &[desc, cost_map]: benchmark_results) { - max_desc_size = std::max(max_desc_size, desc.size()); + size_t max_expr_size = 0; + for (const auto &[expr, cost_list]: benchmark_results) { + max_expr_size = std::max(max_expr_size, expr.size()); } - for (const auto &[desc, cost_map]: benchmark_results) { - for (size_t i = 0; i < max_desc_size - desc.size(); ++i) { + for (const auto &[expr, cost_list]: benchmark_results) { + for (size_t i = 0; i < max_expr_size - expr.size(); ++i) { fprintf(stderr, " "); } - fprintf(stderr, "%s: ", desc.c_str()); + fprintf(stderr, "%s: ", expr.c_str()); size_t cnt = 0; double baseline_cost = 0.0; double with_universal_cost = 0.0; double universal_only_cost = 0.0; - for (const auto &[name, cost]: cost_map) { + for (const auto &[name, cost]: cost_list) { if (++cnt > 1) { fprintf(stderr, ", "); } @@ -336,7 +342,7 @@ int main(int argc, char **argv) { --argc; } if ((argc > 1) && (slow_option == argv[1])) { - budget = 5.0; + budget = 10.0; ++argv; --argc; } diff --git a/eval/src/vespa/eval/instruction/sparse_join_reduce_plan.cpp b/eval/src/vespa/eval/instruction/sparse_join_reduce_plan.cpp index 00499e7f997..fbef6ee5b7f 100644 --- a/eval/src/vespa/eval/instruction/sparse_join_reduce_plan.cpp +++ b/eval/src/vespa/eval/instruction/sparse_join_reduce_plan.cpp @@ -38,149 +38,80 @@ size_t count_only_in_second(const Dims &first, const Dims &second) { return result; } -struct SparseJoinReduceState { - SmallVector<string_id,4> addr_space; - SmallVector<string_id*,4> a_addr; - SmallVector<const string_id*,4> overlap; - SmallVector<string_id*,4> b_only; - SmallVector<size_t,4> b_view; - size_t a_subspace; - size_t b_subspace; - uint32_t res_dims; - SparseJoinReduceState(const bool *in_a, const bool *in_b, const bool *in_res, size_t dims) - : addr_space(dims), a_addr(), overlap(), b_only(), b_view(), a_subspace(), b_subspace(), res_dims(0) - { - size_t b_idx = 0; - uint32_t dims_end = addr_space.size(); - for (size_t i = 0; i < dims; ++i) { - string_id *id = in_res[i] ? &addr_space[res_dims++] : &addr_space[--dims_end]; - if (in_a[i]) { - a_addr.push_back(id); - if (in_b[i]) { - overlap.push_back(id); - b_view.push_back(b_idx++); - } - } else if (in_b[i]) { - b_only.push_back(id); - ++b_idx; - } - } - // Kept dimensions are allocated from the start and dropped - // dimensions are allocated from the end. Make sure they - // combine to exactly cover the complete address space. - assert(res_dims == dims_end); - } - ~SparseJoinReduceState(); -}; -SparseJoinReduceState::~SparseJoinReduceState() = default; - -void execute_plan(const Value::Index &a, const Value::Index &b, - const bool *in_a, const bool *in_b, const bool *in_res, - size_t dims, auto &&f) -{ - SparseJoinReduceState state(in_a, in_b, in_res, dims); - auto outer = a.create_view({}); - auto inner = b.create_view(state.b_view); - outer->lookup({}); - while (outer->next_result(state.a_addr, state.a_subspace)) { - inner->lookup(state.overlap); - while (inner->next_result(state.b_only, state.b_subspace)) { - f(state.a_subspace, state.b_subspace, ConstArrayRef<string_id>{state.addr_space.begin(), state.res_dims}); - } - } -} - -using est_fun = SparseJoinReducePlan::est_fun_t; -using est_filter = std::function<bool(bool, bool, bool)>; - -struct Est { - est_filter filter; - est_fun estimate; - bool can_use; - Est(est_filter filter_in, est_fun estimate_in) - : filter(filter_in), estimate(estimate_in), can_use(true) {} - ~Est(); -}; -Est::~Est() = default; - size_t est_1(size_t, size_t) noexcept { return 1; } size_t est_a_or_0(size_t a, size_t b) noexcept { return (b == 0) ? 0 : a; } size_t est_b_or_0(size_t a, size_t b) noexcept { return (a == 0) ? 0 : b; } size_t est_min(size_t a, size_t b) noexcept { return std::min(a, b); } size_t est_mul(size_t a, size_t b) noexcept { return (a * b); } -bool no_dims(bool, bool, bool) noexcept { return false; } bool reduce_all(bool, bool, bool keep) noexcept { return !keep; } -bool keep_a_reduce_b(bool a, bool b, bool keep) noexcept { - if (keep) { - return (a && !b); - } else { - return (!a && b); - } -} -bool keep_b_reduce_a(bool a, bool b, bool keep) noexcept { return keep_a_reduce_b(b, a, keep); } -bool full_overlap(bool a, bool b, bool) noexcept { return (a == b); } +bool keep_a_reduce_b(bool a, bool b, bool keep) noexcept { return (keep == a) && (keep != b); } +bool keep_b_reduce_a(bool a, bool b, bool keep) noexcept { return (keep == b) && (keep != a); } bool no_overlap_keep_all(bool a, bool b, bool keep) noexcept { return keep && (a != b); } -std::vector<Est> make_est_list() { - return { - { no_dims, est_1 }, - { reduce_all, est_1 }, - { keep_a_reduce_b, est_a_or_0 }, - { keep_b_reduce_a, est_b_or_0 }, - { full_overlap, est_min }, - { no_overlap_keep_all, est_mul } - }; -} +} // <unnamed> -void update_est_list(std::vector<Est> &est_list, bool in_lhs, bool in_rhs, bool in_res) { - for (Est &est: est_list) { - if (est.can_use && !est.filter(in_lhs, in_rhs, in_res)) { - est.can_use = false; - } - } +SparseJoinReducePlan::est_fun_t +SparseJoinReducePlan::select_estimate() const +{ + if (check(reduce_all)) return est_1; + if (check(no_overlap_keep_all)) return est_mul; + if (check(keep_a_reduce_b)) return est_a_or_0; + if (check(keep_b_reduce_a)) return est_b_or_0; + return est_min; } -est_fun select_estimate(const std::vector<Est> &est_list) { - for (const Est &est: est_list) { - if (est.can_use) { - return est.estimate; +SparseJoinReducePlan::State::State(const bool *in_a, const bool *in_b, const bool *in_res, size_t dims) + : addr_space(dims), a_addr(), overlap(), b_only(), b_view(), a_subspace(), b_subspace(), res_dims(0) +{ + size_t b_idx = 0; + uint32_t dims_end = addr_space.size(); + for (size_t i = 0; i < dims; ++i) { + string_id *id = in_res[i] ? &addr_space[res_dims++] : &addr_space[--dims_end]; + if (in_a[i]) { + a_addr.push_back(id); + if (in_b[i]) { + overlap.push_back(id); + b_view.push_back(b_idx++); + } + } else if (in_b[i]) { + b_only.push_back(id); + ++b_idx; } } - return est_min; + // Kept dimensions are allocated from the start and dropped + // dimensions are allocated from the end. Make sure they + // combine to exactly cover the complete address space. + assert(res_dims == dims_end); } -} // <unnamed> +SparseJoinReducePlan::State::~State() = default; SparseJoinReducePlan::SparseJoinReducePlan(const ValueType &lhs, const ValueType &rhs, const ValueType &res) - : _in_lhs(), _in_rhs(), _in_res(), _res_dims(0), _estimate() + : _in_lhs(), _in_rhs(), _in_res(), _res_dims(res.count_mapped_dimensions()), _estimate() { auto dims = merge(lhs.mapped_dimensions(), rhs.mapped_dimensions()); assert(count_only_in_second(dims, res.mapped_dimensions()) == 0); - auto est_list = make_est_list(); for (const auto &dim: dims) { _in_lhs.push_back(lhs.has_dimension(dim.name)); _in_rhs.push_back(rhs.has_dimension(dim.name)); _in_res.push_back(res.has_dimension(dim.name)); - if (_in_res.back()) { - ++_res_dims; - } - update_est_list(est_list, _in_lhs.back(), _in_rhs.back(), _in_res.back()); } - _estimate = select_estimate(est_list); - assert(bool(_estimate)); + _estimate = select_estimate(); } SparseJoinReducePlan::~SparseJoinReducePlan() = default; -void -SparseJoinReducePlan::execute(const Value::Index &lhs, const Value::Index &rhs, F f) const { - if (rhs.size() < lhs.size()) { - auto swap = [&](auto a, auto b, auto addr) { f(b, a, addr); }; - execute_plan(rhs, lhs, _in_rhs.data(), _in_lhs.data(), _in_res.data(), _in_res.size(), swap); - } else { - execute_plan(lhs, rhs, _in_lhs.data(), _in_rhs.data(), _in_res.data(), _in_res.size(), f); - } +bool +SparseJoinReducePlan::maybe_forward_lhs_index() const +{ + return check(keep_a_reduce_b); +} + +bool +SparseJoinReducePlan::maybe_forward_rhs_index() const +{ + return check(keep_b_reduce_a); } } // namespace diff --git a/eval/src/vespa/eval/instruction/sparse_join_reduce_plan.h b/eval/src/vespa/eval/instruction/sparse_join_reduce_plan.h index c93bf46e2dc..75b8d329763 100644 --- a/eval/src/vespa/eval/instruction/sparse_join_reduce_plan.h +++ b/eval/src/vespa/eval/instruction/sparse_join_reduce_plan.h @@ -14,8 +14,7 @@ public: friend class SparseJoinReducePlanTest; using BitList = SmallVector<bool,8>; - using est_fun_t = std::function<size_t(size_t lhs_size, size_t rhs_size)>; - using F = std::function<void(size_t lhs_subspace, size_t rhs_subspace, ConstArrayRef<string_id> res_addr)>; + using est_fun_t = size_t (*)(size_t lhs_size, size_t rhs_size) noexcept; private: BitList _in_lhs; @@ -24,15 +23,65 @@ private: size_t _res_dims; est_fun_t _estimate; + struct State { + SmallVector<string_id,4> addr_space; + SmallVector<string_id*,4> a_addr; + SmallVector<const string_id*,4> overlap; + SmallVector<string_id*,4> b_only; + SmallVector<size_t,4> b_view; + size_t a_subspace; + size_t b_subspace; + uint32_t res_dims; + State(const bool *in_a, const bool *in_b, const bool *in_res, size_t dims); + ~State(); + }; + + static void execute_plan(const Value::Index &a, const Value::Index &b, + const bool *in_a, const bool *in_b, const bool *in_res, + size_t dims, auto &&f) + { + State state(in_a, in_b, in_res, dims); + auto outer = a.create_view({}); + auto inner = b.create_view(state.b_view); + outer->lookup({}); + while (outer->next_result(state.a_addr, state.a_subspace)) { + inner->lookup(state.overlap); + while (inner->next_result(state.b_only, state.b_subspace)) { + f(state.a_subspace, state.b_subspace, ConstArrayRef<string_id>{state.addr_space.begin(), state.res_dims}); + } + } + } + + bool check(auto &&pred) const { + for (size_t i = 0; i < _in_lhs.size(); ++i) { + if (!pred(_in_lhs[i], _in_rhs[i], _in_res[i])) { + return false; + } + } + return true; + } + + est_fun_t select_estimate() const; + public: SparseJoinReducePlan(const ValueType &lhs, const ValueType &rhs, const ValueType &res); ~SparseJoinReducePlan(); size_t res_dims() const { return _res_dims; } - bool distinct_result() const { return (_res_dims == _in_res.size()); } + bool distinct_result() const { return _res_dims == _in_res.size(); } + bool maybe_forward_lhs_index() const; + bool maybe_forward_rhs_index() const; size_t estimate_result_size(const Value::Index &lhs, const Value::Index &rhs) const { return _estimate(lhs.size(), rhs.size()); } - void execute(const Value::Index &lhs, const Value::Index &rhs, F f) const; + // f ~= std::function<void(size_t lhs_subspace, size_t rhs_subspace, ConstArrayRef<string_id> res_addr)>; + void execute(const Value::Index &lhs, const Value::Index &rhs, auto &&f) const { + if (rhs.size() < lhs.size()) { + auto swap = [&f](auto a, auto b, auto addr) { f(b, a, addr); }; + execute_plan(rhs, lhs, _in_rhs.data(), _in_lhs.data(), _in_res.data(), _in_res.size(), swap); + } else { + execute_plan(lhs, rhs, _in_lhs.data(), _in_rhs.data(), _in_res.data(), _in_res.size(), f); + } + } }; } // namespace diff --git a/eval/src/vespa/eval/instruction/universal_dot_product.cpp b/eval/src/vespa/eval/instruction/universal_dot_product.cpp index 86e6be52de4..3811508a543 100644 --- a/eval/src/vespa/eval/instruction/universal_dot_product.cpp +++ b/eval/src/vespa/eval/instruction/universal_dot_product.cpp @@ -74,22 +74,58 @@ void my_universal_dot_product_op(InterpretedFunction::State &state, uint64_t par state.pop_pop_push(result); } +template <typename LCT, typename RCT, typename OCT> +void my_universal_dense_dot_product_op(InterpretedFunction::State &state, uint64_t param_in) { + using dot_product = DotProduct<LCT,RCT>; + const auto ¶m = unwrap_param<UniversalDotProductParam>(param_in); + const auto &lhs = state.peek(1); + const auto &rhs = state.peek(0); + size_t lhs_index_size = lhs.index().size(); + size_t rhs_index_size = rhs.index().size(); + if (rhs_index_size == 0 || lhs_index_size == 0) { + const Value &empty = state.stash.create<ValueView>(param.res_type, EmptyIndex::get(), TypedCells(nullptr, get_cell_type<OCT>(), 0)); + state.pop_pop_push(empty); + return; + } + const auto lhs_cells = lhs.cells().typify<LCT>(); + const auto rhs_cells = rhs.cells().typify<RCT>(); + auto dst_cells = state.stash.create_array<OCT>(lhs_index_size * param.dense_plan.res_size); + auto dense_fun = [&](size_t lhs_idx, size_t rhs_idx, size_t dst_idx) { + dst_cells[dst_idx] += dot_product::apply(&lhs_cells[lhs_idx], &rhs_cells[rhs_idx], param.vector_size); + }; + for (size_t lhs_subspace = 0; lhs_subspace < lhs_index_size; ++lhs_subspace) { + for (size_t rhs_subspace = 0; rhs_subspace < rhs_index_size; ++rhs_subspace) { + param.dense_plan.execute(lhs_subspace * param.dense_plan.lhs_size, + rhs_subspace * param.dense_plan.rhs_size, + lhs_subspace * param.dense_plan.res_size, dense_fun); + } + } + const Value &result = state.stash.create<ValueView>(param.res_type, lhs.index(), TypedCells(dst_cells)); + state.pop_pop_push(result); +} + struct SelectUniversalDotProduct { - template <typename LCM, typename RCM, typename SCALAR> static auto invoke(const UniversalDotProductParam &) { + template <typename LCM, typename RCM, typename SCALAR> static auto invoke(const UniversalDotProductParam ¶m) { constexpr CellMeta ocm = CellMeta::join(LCM::value, RCM::value).reduce(SCALAR::value); using LCT = CellValueType<LCM::value.cell_type>; using RCT = CellValueType<RCM::value.cell_type>; using OCT = CellValueType<ocm.cell_type>; + if (param.sparse_plan.maybe_forward_lhs_index()) { + return my_universal_dense_dot_product_op<LCT,RCT,OCT>; + } return my_universal_dot_product_op<LCT,RCT,OCT>; } }; bool check_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs) { - UniversalDotProductParam param(res, lhs, rhs); - if (param.vector_size < 8) { + (void) res; + if (lhs.is_double() || rhs.is_double()) { return false; } - return true; + if (lhs.count_mapped_dimensions() > 0 || rhs.count_mapped_dimensions() > 0) { + return true; + } + return false; } } // namespace <unnamed> @@ -118,8 +154,15 @@ UniversalDotProduct::optimize(const TensorFunction &expr, Stash &stash, bool for { if (auto reduce = as<Reduce>(expr); reduce && (reduce->aggr() == Aggr::SUM)) { if (auto join = as<Join>(reduce->child()); join && (join->function() == Mul::f)) { - if (force || check_types(expr.result_type(), join->lhs().result_type(), join->rhs().result_type())) { - return stash.create<UniversalDotProduct>(expr.result_type(), join->lhs(), join->rhs()); + const ValueType &res_type = expr.result_type(); + const ValueType &lhs_type = join->lhs().result_type(); + const ValueType &rhs_type = join->rhs().result_type(); + if (force || check_types(res_type, lhs_type, rhs_type)) { + SparseJoinReducePlan sparse_plan(lhs_type, rhs_type, res_type); + if (sparse_plan.maybe_forward_rhs_index() && !sparse_plan.maybe_forward_lhs_index()) { + return stash.create<UniversalDotProduct>(res_type, join->rhs(), join->lhs()); + } + return stash.create<UniversalDotProduct>(res_type, join->lhs(), join->rhs()); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index a67a513550a..ac13ee992c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -116,8 +116,11 @@ public class NodeRepositoryProvisioner implements Provisioner { nodeRepository.nodeResourceLimits().ensureWithinAdvertisedLimits("Min", requested.minResources().nodeResources(), application, cluster); nodeRepository.nodeResourceLimits().ensureWithinAdvertisedLimits("Max", requested.maxResources().nodeResources(), application, cluster); - if ( ! requested.minResources().nodeResources().gpuResources().equals(requested.maxResources().nodeResources().gpuResources())) - throw new IllegalArgumentException(requested + " is invalid: Gpu capacity cannot have ranges"); + if (!requested.minResources().nodeResources().gpuResources().equals(requested.maxResources().nodeResources().gpuResources())) + throw new IllegalArgumentException(requested + " is invalid: GPU capacity cannot have ranges"); + + if (!requested.minResources().nodeResources().gpuResources().isZero() && !zone.system().isPublic()) + throw new IllegalArgumentException(requested + " is invalid: GPUs are not supported in " + zone); logInsufficientDiskResources(cluster, requested, logger); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 5539bb0cb6e..38b8836188b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.Architecture; @@ -502,10 +501,16 @@ public class DynamicProvisioningTest { public void gpu_host() { List<Flavor> flavors = List.of(new Flavor("gpu", new NodeResources(4, 16, 125, 10, fast, local, Architecture.x86_64, new NodeResources.GpuResources(1, 16)))); - ProvisioningTester tester = new ProvisioningTester.Builder().dynamicProvisioning(true, false) - .flavors(flavors) + ProvisioningTester tester = new ProvisioningTester.Builder().flavors(flavors) .hostProvisioner(new MockHostProvisioner(flavors)) .nameResolver(nameResolver) + .zone(new Zone(Cloud.builder() + .dynamicProvisioning(true) + .allowHostSharing(false) + .build(), + SystemName.Public, + Environment.defaultEnvironment(), + RegionName.defaultName())) .build(); NodeResources resources = new NodeResources(4, 16, 125, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any, diff --git a/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt b/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt index 704fe238ed5..f2161196c32 100644 --- a/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt @@ -9,3 +9,6 @@ vespa_add_library(searchcommon_searchcommon_attribute OBJECT status.cpp DEPENDS ) + +file(GLOB HEADERS *.h) +install(FILES ${HEADERS} DESTINATION include/vespa/searchcommon/attribute) diff --git a/searchlib/src/vespa/searchcommon/common/CMakeLists.txt b/searchlib/src/vespa/searchcommon/common/CMakeLists.txt index 6cc02ae7884..67adea7ea96 100644 --- a/searchlib/src/vespa/searchcommon/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchcommon/common/CMakeLists.txt @@ -8,3 +8,6 @@ vespa_add_library(searchcommon_searchcommon_common OBJECT schemaconfigurer.cpp DEPENDS ) + +file(GLOB HEADERS *.h) +install(FILES ${HEADERS} DESTINATION include/vespa/searchcommon/common) diff --git a/storage/src/tests/distributor/pendingmessagetrackertest.cpp b/storage/src/tests/distributor/pendingmessagetrackertest.cpp index 8277281206d..35dc072b953 100644 --- a/storage/src/tests/distributor/pendingmessagetrackertest.cpp +++ b/storage/src/tests/distributor/pendingmessagetrackertest.cpp @@ -1,19 +1,24 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/dummystoragelink.h> #include <vespa/document/base/testdocman.h> -#include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> -#include <tests/common/dummystoragelink.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <functional> +#include <vector> +using document::Bucket; +using document::BucketId; using document::test::makeDocumentBucket; using document::test::makeBucketSpace; using namespace ::testing; @@ -23,8 +28,7 @@ namespace storage::distributor { using namespace std::chrono_literals; struct PendingMessageTrackerTest : Test { - void insertMessages(PendingMessageTracker& tracker); - + std::vector<std::shared_ptr<api::StorageCommand>> insertMessages(PendingMessageTracker& tracker); }; namespace { @@ -33,11 +37,7 @@ class RequestBuilder { uint16_t _toNode; std::chrono::milliseconds _atTime; public: - RequestBuilder() - : _toNode(0), - _atTime() - { - } + RequestBuilder() noexcept : _toNode(0), _atTime() {} RequestBuilder& atTime(std::chrono::milliseconds t) { _atTime = t; @@ -59,14 +59,12 @@ makeStorageAddress(uint16_t node) { return {&_storage, lib::NodeType::STORAGE, node}; } -class Fixture -{ - StorageComponentRegisterImpl _compReg; +class Fixture { + StorageComponentRegisterImpl _compReg; framework::defaultimplementation::FakeClock _clock; - std::unique_ptr<PendingMessageTracker> _tracker; - document::TestDocMan _testDocMan; + std::unique_ptr<PendingMessageTracker> _tracker; + document::TestDocMan _testDocMan; public: - Fixture(); ~Fixture(); @@ -110,21 +108,26 @@ public: PendingMessageTracker& tracker() { return *_tracker; } auto& clock() { return _clock; } + std::vector<uint64_t> enumerate_msg_ids(const std::function<bool(const document::Bucket&)>& bucket_predicate) const { + std::vector<uint64_t> enumerated_ids; + auto insert_enumerated_ids = [&](uint64_t msg_id) { enumerated_ids.emplace_back(msg_id); }; + + _tracker->enumerate_matching_pending_bucket_ops(bucket_predicate, insert_enumerated_ids); + return enumerated_ids; + } + private: - std::string createDummyIdString(const document::BucketId& bucket) const { + static std::string createDummyIdString(const document::BucketId& bucket) { std::ostringstream id; id << "id:foo:testdoctype1:n=" << bucket.getId() << ":foo"; return id.str(); } - document::Document::SP createDummyDocumentForBucket(const document::BucketId& bucket) const - { + document::Document::SP createDummyDocumentForBucket(const document::BucketId& bucket) const { return _testDocMan.createDocument("foobar", createDummyIdString(bucket)); } - std::shared_ptr<api::RemoveCommand> createRemoveToNode( - uint16_t node) const - { + std::shared_ptr<api::RemoveCommand> createRemoveToNode(uint16_t node) const { document::BucketId bucket(16, 1234); auto cmd = std::make_shared<api::RemoveCommand>( makeDocumentBucket(bucket), @@ -195,9 +198,10 @@ TEST_F(PendingMessageTrackerTest, simple) { } } -void +std::vector<std::shared_ptr<api::StorageCommand>> PendingMessageTrackerTest::insertMessages(PendingMessageTracker& tracker) { + std::vector<std::shared_ptr<api::StorageCommand>> inserted; for (uint32_t i = 0; i < 4; i++) { std::ostringstream ost; ost << "id:footype:testdoc:n=1234:" << i; @@ -206,15 +210,19 @@ PendingMessageTrackerTest::insertMessages(PendingMessageTracker& tracker) document::DocumentId(ost.str()), 1000 + i); remove->setAddress(makeStorageAddress(i % 2)); tracker.insert(remove); + inserted.emplace_back(std::move(remove)); } for (uint32_t i = 0; i < 4; i++) { std::ostringstream ost; ost << "id:footype:testdoc:n=4567:" << i; - auto remove = std::make_shared<api::RemoveCommand>(makeDocumentBucket(document::BucketId(16, 4567)), document::DocumentId(ost.str()), 2000 + i); + auto remove = std::make_shared<api::RemoveCommand>(makeDocumentBucket(document::BucketId(16, 4567)), + document::DocumentId(ost.str()), 2000 + i); remove->setAddress(makeStorageAddress(i % 2)); tracker.insert(remove); + inserted.emplace_back(std::move(remove)); } + return inserted; } TEST_F(PendingMessageTrackerTest, start_page) { @@ -291,15 +299,13 @@ TEST_F(PendingMessageTrackerTest, multiple_messages) { namespace { -class TestChecker : public PendingMessageTracker::Checker -{ +class TestChecker : public PendingMessageTracker::Checker { public: uint8_t pri; - TestChecker() : pri(UINT8_MAX) {} + TestChecker() noexcept : pri(UINT8_MAX) {} - bool check(uint32_t msgType, uint16_t node, uint8_t p) override { - (void) node; + bool check(uint32_t msgType, [[maybe_unused]] uint16_t node, uint8_t p) override { if (msgType == api::MessageType::REMOVE_ID) { pri = p; return false; @@ -309,7 +315,6 @@ public: } }; - } TEST_F(PendingMessageTrackerTest, get_pending_message_types) { @@ -370,12 +375,10 @@ TEST_F(PendingMessageTrackerTest, has_pending_message) { namespace { -class OperationEnumerator : public PendingMessageTracker::Checker -{ +class OperationEnumerator : public PendingMessageTracker::Checker { std::ostringstream ss; public: - bool check(uint32_t msgType, uint16_t node, uint8_t p) override { - (void) p; + bool check(uint32_t msgType, uint16_t node, [[maybe_unused]] uint8_t p) override { ss << api::MessageType::get(static_cast<api::MessageType::Id>(msgType)).getName() << " -> " << node << "\n"; @@ -574,4 +577,58 @@ TEST_F(PendingMessageTrackerTest, request_bucket_info_with_bucket_tracked_with_s } } +namespace { + +auto bucket_id_eq_fn(BucketId matched_id) { + return [matched_id](auto& bucket) noexcept { + return bucket.getBucketId() == matched_id; + }; +} + +auto bucket_eq_fn(Bucket matched_bucket) { + return [matched_bucket](auto& bucket) noexcept { + return bucket == matched_bucket; + }; +} + +} + +TEST_F(PendingMessageTrackerTest, can_enumerate_all_message_ids_for_ops_to_matching_buckets) { + Fixture f; + BucketId bucket_id_a(16, 1234); + BucketId bucket_id_b(16, 4567); + // This inserts 4 ops for bucket {16, 1234} (bucket 'a') and 4 ops for {16, 4567} (bucket 'b') + auto inserted_ops = insertMessages(f.tracker()); + std::vector<uint64_t> bucket_a_msgs_ids, bucket_b_msgs_ids; + for (auto& op : inserted_ops) { + if (op->getBucketId() == bucket_id_a) { + bucket_a_msgs_ids.emplace_back(op->getMsgId()); + } else { + bucket_b_msgs_ids.emplace_back(op->getMsgId()); + } + } + + // Match all for bucket 'a' + EXPECT_THAT(f.enumerate_msg_ids(bucket_id_eq_fn(bucket_id_a)), UnorderedElementsAreArray(bucket_a_msgs_ids)); + // Match all for bucket 'b' + EXPECT_THAT(f.enumerate_msg_ids(bucket_id_eq_fn(bucket_id_b)), UnorderedElementsAreArray(bucket_b_msgs_ids)); + // Match no buckets + EXPECT_THAT(f.enumerate_msg_ids(bucket_id_eq_fn(BucketId(16, 7890))), IsEmpty()); + + const auto default_space = document::FixedBucketSpaces::default_space(); + const auto global_space = document::FixedBucketSpaces::global_space(); + + // Message to global bucket space (the former messages were all in the default space) + auto global_msg = std::make_shared<api::RequestBucketInfoCommand>(global_space, std::vector<BucketId>({bucket_id_a})); + global_msg->setAddress(makeStorageAddress(3)); + f.tracker().insert(global_msg); + + // Default space has the expected existing entries + EXPECT_THAT(f.enumerate_msg_ids(bucket_eq_fn(Bucket(default_space, bucket_id_a))), + UnorderedElementsAreArray(bucket_a_msgs_ids)); + // Global space has only 1 entry + EXPECT_THAT(f.enumerate_msg_ids(bucket_eq_fn(Bucket(global_space, bucket_id_a))), + ElementsAre(global_msg->getMsgId())); +} + } diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp index 7b3cdacf702..c32b3b83c05 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp @@ -40,7 +40,8 @@ PendingMessageTracker::MessageEntry::toHtml() const { vespalib::asciistream ss; ss << "<li><i>Node " << nodeIdx << "</i>: " << "<b>" << vespalib::to_string(timeStamp) << "</b> " - << api::MessageType::get(api::MessageType::Id(msgType)).getName() << "(" << bucket.getBucketId() << ", priority=" << priority << ")</li>\n"; + << api::MessageType::get(api::MessageType::Id(msgType)).getName() + << "(" << bucket.getBucketId() << ", priority=" << priority << ")</li>\n"; return ss.str(); } @@ -83,7 +84,7 @@ std::vector<uint64_t> PendingMessageTracker::clearMessagesForNode(uint16_t node) { std::lock_guard guard(_lock); - MessagesByNodeAndBucket& idx(boost::multi_index::get<1>(_messages)); + auto& idx = boost::multi_index::get<IndexByNodeAndBucket>(_messages); auto range = pairAsRange(idx.equal_range(boost::make_tuple(node))); std::vector<uint64_t> erasedIds; @@ -97,6 +98,27 @@ PendingMessageTracker::clearMessagesForNode(uint16_t node) } void +PendingMessageTracker::enumerate_matching_pending_bucket_ops( + const std::function<bool(const document::Bucket&)>& bucket_predicate, + const std::function<void(uint64_t)>& msg_id_callback) const +{ + std::lock_guard guard(_lock); + const auto& idx = boost::multi_index::get<IndexByBucketAndType>(_messages); + auto iter = idx.begin(); + const auto last = idx.end(); + while (iter != last) { + const auto check_bucket = iter->bucket; + const bool match = bucket_predicate(check_bucket); + do { + if (match) { + msg_id_callback(iter->msgId); + } + ++iter; + } while ((iter != last) && (iter->bucket == check_bucket)); + } +} + +void PendingMessageTracker::insert(const std::shared_ptr<api::StorageMessage>& msg) { if (msg->getAddress()) { @@ -126,8 +148,8 @@ PendingMessageTracker::reply(const api::StorageReply& r) uint64_t msgId = r.getMsgId(); std::unique_lock guard(_lock); - MessagesByMsgId& msgs = boost::multi_index::get<0>(_messages); - MessagesByMsgId::iterator iter = msgs.find(msgId); + auto& msgs = boost::multi_index::get<IndexByMessageId>(_messages); + auto iter = msgs.find(msgId); if (iter != msgs.end()) { bucket = iter->bucket; _nodeInfo.decPending(r.getAddress()->getIndex()); @@ -184,7 +206,7 @@ bool range_is_empty_or_only_has_read_ops(const Range& range) noexcept { bool PendingMessageTracker::bucket_has_no_pending_write_ops(const document::Bucket& bucket) const noexcept { - auto& bucket_idx = boost::multi_index::get<2>(_messages); + auto& bucket_idx = boost::multi_index::get<IndexByBucketAndType>(_messages); auto pending_tasks_for_bucket = bucket_idx.equal_range(bucket); return range_is_empty_or_only_has_read_ops(pending_tasks_for_bucket); } @@ -243,30 +265,30 @@ runCheckerOnRange(PendingMessageTracker::Checker& checker, const Range& range) } void -PendingMessageTracker::checkPendingMessages(uint16_t node, const document::Bucket &bucket, Checker& checker) const +PendingMessageTracker::checkPendingMessages(uint16_t node, const document::Bucket& bucket, Checker& checker) const { std::lock_guard guard(_lock); - const MessagesByNodeAndBucket& msgs(boost::multi_index::get<1>(_messages)); + const auto& msgs = boost::multi_index::get<IndexByNodeAndBucket>(_messages); auto range = pairAsRange(msgs.equal_range(boost::make_tuple(node, bucket))); runCheckerOnRange(checker, range); } void -PendingMessageTracker::checkPendingMessages(const document::Bucket &bucket, Checker& checker) const +PendingMessageTracker::checkPendingMessages(const document::Bucket& bucket, Checker& checker) const { std::lock_guard guard(_lock); - const MessagesByBucketAndType& msgs(boost::multi_index::get<2>(_messages)); + const auto& msgs = boost::multi_index::get<IndexByBucketAndType>(_messages); auto range = pairAsRange(msgs.equal_range(boost::make_tuple(bucket))); runCheckerOnRange(checker, range); } bool -PendingMessageTracker::hasPendingMessage(uint16_t node, const document::Bucket &bucket, uint32_t messageType) const +PendingMessageTracker::hasPendingMessage(uint16_t node, const document::Bucket& bucket, uint32_t messageType) const { std::lock_guard guard(_lock); - const MessagesByNodeAndBucket& msgs(boost::multi_index::get<1>(_messages)); + const auto& msgs = boost::multi_index::get<IndexByNodeAndBucket>(_messages); auto range = msgs.equal_range(boost::make_tuple(node, bucket, messageType)); return (range.first != range.second); @@ -283,7 +305,7 @@ void PendingMessageTracker::getStatusPerBucket(std::ostream& out) const { std::lock_guard guard(_lock); - const MessagesByNodeAndBucket& msgs = boost::multi_index::get<1>(_messages); + const auto& msgs = boost::multi_index::get<IndexByNodeAndBucket>(_messages); using BucketMap = std::map<document::Bucket, std::vector<vespalib::string>>; BucketMap perBucketMsgs; for (const auto& msg : msgs) { @@ -312,9 +334,9 @@ void PendingMessageTracker::getStatusPerNode(std::ostream& out) const { std::lock_guard guard(_lock); - const MessagesByNodeAndBucket& msgs = boost::multi_index::get<1>(_messages); + const auto& msgs = boost::multi_index::get<IndexByNodeAndBucket>(_messages); int lastNode = -1; - for (const auto & node : msgs) { + for (const auto& node : msgs) { if (node.nodeIdx != lastNode) { if (lastNode != -1) { out << "</ul>\n"; diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.h b/storage/src/vespa/storage/distributor/pendingmessagetracker.h index 4b5655d3f3c..736f2918401 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.h +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.h @@ -2,22 +2,23 @@ #pragma once #include "nodeinfo.h" -#include <vespa/storageframework/generic/status/htmlstatusreporter.h> -#include <vespa/storageframework/generic/component/componentregister.h> -#include <vespa/storageframework/generic/component/component.h> #include <vespa/storageapi/message/bucket.h> +#include <vespa/storageframework/generic/component/component.h> +#include <vespa/storageframework/generic/component/componentregister.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/vespalib/stllike/hash_set.h> -#include <boost/multi_index_container.hpp> +#include <boost/multi_index/composite_key.hpp> #include <boost/multi_index/identity.hpp> -#include <boost/multi_index/member.hpp> #include <boost/multi_index/mem_fun.hpp> +#include <boost/multi_index/member.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> -#include <boost/multi_index/composite_key.hpp> -#include <set> -#include <unordered_map> +#include <boost/multi_index_container.hpp> #include <chrono> +#include <functional> #include <mutex> +#include <set> +#include <unordered_map> namespace storage::distributor { @@ -84,20 +85,20 @@ public: * passing it to the given type checker. * Breaks when the checker returns false. */ - void checkPendingMessages(uint16_t node, const document::Bucket &bucket, Checker& checker) const; + void checkPendingMessages(uint16_t node, const document::Bucket& bucket, Checker& checker) const; /** * Goes through each pending message (across all nodes) for the given bucket * and invokes the given checker with the node, message type and priority. * Breaks when the checker returns false. */ - void checkPendingMessages(const document::Bucket &bucket, Checker& checker) const; + void checkPendingMessages(const document::Bucket& bucket, Checker& checker) const; /** * Utility function for checking if there's a message of type * messageType pending to bucket bid on the given node. */ - bool hasPendingMessage(uint16_t node, const document::Bucket &bucket, uint32_t messageType) const; + bool hasPendingMessage(uint16_t node, const document::Bucket& bucket, uint32_t messageType) const; /** * Returns a vector containing the number of pending messages to each storage node. @@ -119,6 +120,17 @@ public: void run_once_no_pending_for_bucket(const document::Bucket& bucket, std::unique_ptr<DeferredTask> task); void abort_deferred_tasks(); + + /** + * For each distinct bucket with at least one pending message towards it: + * + * Iff `bucket_predicate(bucket) == true`, `msg_id_callback` is invoked once for _each_ + * message towards `bucket`, with the message ID as the argument. + * + * Note: `bucket_predicate` is only invoked once per distinct bucket. + */ + void enumerate_matching_pending_bucket_ops(const std::function<bool(const document::Bucket&)>& bucket_predicate, + const std::function<void(uint64_t)>& msg_id_callback) const; private: struct MessageEntry { TimePoint timeStamp; @@ -169,9 +181,11 @@ private: > >; - using MessagesByMsgId = Messages::nth_index<0>::type; - using MessagesByNodeAndBucket = Messages::nth_index<1>::type; - using MessagesByBucketAndType = Messages::nth_index<2>::type; + // Must match Messages::nth_index<N> + static constexpr uint32_t IndexByMessageId = 0; + static constexpr uint32_t IndexByNodeAndBucket = 1; + static constexpr uint32_t IndexByBucketAndType = 2; + using DeferredBucketTaskMap = std::unordered_multimap< document::Bucket, std::unique_ptr<DeferredTask>, diff --git a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h index 54398c0b0d9..d20cfc07a9a 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h +++ b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/sanitizers.h> +#include <vespa/config.h> #include <algorithm> #include <array> #include <cassert> |