diff options
42 files changed, 672 insertions, 334 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index f77b40bc67e..4a345a1f0ec 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -14,6 +14,9 @@ public enum NodeType { proxy, /** A node to be assigned to a tenant to run application workloads */ - tenant + tenant, + + /** A config server */ + config } diff --git a/cppunit-parallelize.py b/cppunit-parallelize.py index 25106568d56..058ab61b2a0 100755 --- a/cppunit-parallelize.py +++ b/cppunit-parallelize.py @@ -7,6 +7,7 @@ import copy import os import subprocess import time +import shlex def parse_arguments(): argparser = argparse.ArgumentParser(description="Run Vespa cppunit tests in parallell") @@ -14,7 +15,7 @@ def parse_arguments(): argparser.add_argument("--chunks", type=int, help="Number of chunks", default=5) args = argparser.parse_args() if args.chunks < 1: - raise RuntimeError, "I require at least one chunk" + raise RuntimeError("Error: Chunk size must be greater than 0") return args @@ -32,6 +33,20 @@ def chunkify(lst, chunks): return result +def error_if_file_not_found(function): + def wrapper(*args, **kwargs): + try: + return function(*args, **kwargs) + except OSError as e: + if e.errno == os.errno.ENOENT: # "No such file or directory" + print >>sys.stderr, "Error: could not find testrunner or valgrind executable" + sys.exit(1) + return wrapper + +@error_if_file_not_found +def get_test_suites(testrunner): + return subprocess.check_output((testrunner, "--list")).strip().split("\n") + class Process: def __init__(self, cmd, group): self.group = group @@ -43,13 +58,14 @@ class Process: stderr=subprocess.STDOUT, preexec_fn=os.setpgrp) +@error_if_file_not_found def build_processes(test_groups): valgrind = os.getenv("VALGRIND") - testrunner = (valgrind, args.testrunner) if valgrind else (args.testrunner,) + testrunner = shlex.split(valgrind) + [args.testrunner] if valgrind else [args.testrunner] processes = [] for group in test_groups: - cmd = testrunner + tuple(group) + cmd = testrunner + group processes.append(Process(cmd, group)) return processes @@ -63,8 +79,8 @@ def cleanup_processes(processes): print >>sys.stderr, e.message args = parse_arguments() -test_suites = subprocess.check_output((args.testrunner, "--list")).strip().split("\n") -test_suite_groups = chunkify(test_suites, min(len(test_suites), args.chunks)) +test_suites = get_test_suites(args.testrunner) +test_suite_groups = chunkify(test_suites, args.chunks) processes = build_processes(test_suite_groups) print "Running %d test suites in %d parallel chunks with ~%d tests each" % (len(test_suites), len(test_suite_groups), len(test_suite_groups[0])) @@ -85,7 +101,7 @@ while True: print "All test suites ran successfully" sys.exit(0) elif return_code is not None: - print "One of '%s' test suites failed:" % ", ".join(proc.group) + print "Error: one of '%s' test suites failed:" % ", ".join(proc.group) print >>sys.stderr, proc.output sys.exit(return_code) diff --git a/document/src/tests/CMakeLists.txt b/document/src/tests/CMakeLists.txt index 83eb2571626..df024925199 100644 --- a/document/src/tests/CMakeLists.txt +++ b/document/src/tests/CMakeLists.txt @@ -34,8 +34,9 @@ vespa_add_executable(document_testrunner_app TEST document_documentconfig ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME document_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:document_testrunner_app> + DEPENDS document_testrunner_app ) diff --git a/memfilepersistence/src/tests/CMakeLists.txt b/memfilepersistence/src/tests/CMakeLists.txt index 079a8036e95..fdb1564d7f9 100644 --- a/memfilepersistence/src/tests/CMakeLists.txt +++ b/memfilepersistence/src/tests/CMakeLists.txt @@ -10,8 +10,9 @@ vespa_add_executable(memfilepersistence_testrunner_app TEST memfilepersistence_testtools ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME memfilepersistence_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:memfilepersistence_testrunner_app> + DEPENDS memfilepersistence_testrunner_app ) diff --git a/metrics/src/tests/CMakeLists.txt b/metrics/src/tests/CMakeLists.txt index 27dd72d3aa8..fdc2f290be8 100644 --- a/metrics/src/tests/CMakeLists.txt +++ b/metrics/src/tests/CMakeLists.txt @@ -16,8 +16,9 @@ vespa_add_executable(metrics_testrunner_app TEST vdstestlib ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME metrics_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:metrics_testrunner_app> + DEPENDS metrics_testrunner_app ) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 875605ff93a..9a7b5c62119 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import java.time.Clock; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -58,6 +59,8 @@ import java.util.stream.Collectors; public class NodeRepository extends AbstractComponent { private final CuratorDatabaseClient zkClient; + private final Curator curator; + private final NodeFlavors flavors; private final NameResolver nameResolver; /** @@ -75,6 +78,8 @@ public class NodeRepository extends AbstractComponent { */ public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver) { this.zkClient = new CuratorDatabaseClient(flavors, curator, clock, zone, nameResolver); + this.curator = curator; + this.flavors = flavors; this.nameResolver = nameResolver; // read and write all nodes to make sure they are stored in the latest version of the serialized format @@ -118,6 +123,33 @@ public class NodeRepository extends AbstractComponent { public List<Node> getInactive() { return zkClient.getNodes(Node.State.inactive); } public List<Node> getFailed() { return zkClient.getNodes(Node.State.failed); } + /** + * Returns a list of nodes that should be trusted by the given node. The list will contain: + * + * - All nodes in the same application as the given node (if node is allocated) + * - All proxy nodes + * - All config servers + */ + public List<Node> getTrustedNodes(Node node) { + final List<Node> trustedNodes = new ArrayList<>(); + + // Add nodes in application + node.allocation().ifPresent(allocation -> trustedNodes.addAll(getNodes(allocation.owner()))); + + // Add proxy nodes + trustedNodes.addAll(getNodes(NodeType.proxy)); + + // Add config servers + // TODO: Revisit this when config servers are added to the repository + Arrays.stream(curator.connectionSpec().split(",")) + .map(hostPort -> hostPort.split(":")[0]) + .map(host -> createNode(host, host, Optional.empty(), flavors.getFlavorOrThrow("default"), + NodeType.config)) + .forEach(trustedNodes::add); + + return Collections.unmodifiableList(trustedNodes); + } + // ----------------- Node lifecycle ----------------------------------------------------------- /** Creates a new node object, without adding it to the node repo */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java index e37f8ccc3f9..c76f5e2455f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java @@ -1,8 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.log.LogLevel; + import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.logging.Logger; /** * Interface for a basic name to IP address resolver. Default implementation delegates to @@ -12,11 +15,14 @@ import java.net.UnknownHostException; */ public interface NameResolver { + Logger log = Logger.getLogger(NameResolver.class.getName()); + /** Resolve IP address from given host name */ default String getByNameOrThrow(String hostname) { try { return InetAddress.getByName(hostname).getHostAddress(); } catch (UnknownHostException e) { + log.log(LogLevel.ERROR, String.format("Failed to resolve hostname %s: %s", hostname, e.getMessage())); throw new RuntimeException(e); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java new file mode 100644 index 00000000000..5112e08d868 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +/** + * @author mpolden + */ +public class NodeAclResponse extends HttpResponse { + + private final NodeRepository nodeRepository; + private final Slime slime; + + public NodeAclResponse(HttpRequest request, NodeRepository nodeRepository) { + super(200); + this.nodeRepository = nodeRepository; + this.slime = new Slime(); + + final Cursor root = slime.setObject(); + final String hostname = baseName(request.getUri().getPath()); + toSlime(hostname, root); + } + + private static String baseName(String path) { + return new File(path).getName(); + } + + private void toSlime(String hostname, Cursor object) { + final Node node = nodeRepository.getNode(hostname) + .orElseThrow(() -> new IllegalArgumentException("No node with hostname '" + hostname + "'")); + toSlime(node, nodeRepository.getTrustedNodes(node), object); + } + + private void toSlime(Node node, List<Node> trustedNodes, Cursor object) { + object.setString("hostname", node.hostname()); + object.setString("ipAddress", node.ipAddress()); + toSlime(trustedNodes, object.setArray("trustedNodes")); + } + + private void toSlime(List<Node> trustedNodes, Cursor array) { + trustedNodes.forEach(node -> { + Cursor object = array.addObject(); + object.setString("hostname", node.hostname()); + object.setString("ipAddress", node.ipAddress()); + }); + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(SlimeUtils.toJsonBytes(slime)); + } + + @Override + public String getContentType() { + return "application/json"; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 69c6112efb2..8832ad0f57f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -86,6 +86,7 @@ public class NodesApiHandler extends LoggingRequestHandler { if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, nodeRepository); if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, nodeRepository); if (path.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, nodeRepository); + if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); return ErrorResponse.notFoundError("Nothing at path '" + request.getUri().getPath() + "'"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index e6901c6e159..f14ccea844b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -39,12 +40,18 @@ public class MockNodeRepository extends NodeRepository { * @param flavors flavors to have in node repo */ public MockNodeRepository(NodeFlavors flavors) throws Exception { - super(flavors, new MockCurator(), Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), + super(flavors, mockCurator(), Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), new MockNameResolver().mockAnyLookup()); this.flavors = flavors; populate(); } + private static Curator mockCurator() { + MockCurator mockCurator = new MockCurator(); + mockCurator.setConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); + return mockCurator; + } + private void populate() { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, flavors, Zone.defaultZone()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 6fc3cbaa596..cc39b295e7b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; @@ -23,16 +22,15 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Duration; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import static org.junit.Assert.assertEquals; @@ -117,7 +115,7 @@ public class FailedExpirerTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); provisioner.prepare(applicationId, cluster, Capacity.fromNodeCount(3), 1, null); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); - provisioner.activate(transaction, applicationId, asHosts(nodes)); + provisioner.activate(transaction, applicationId, ProvisioningTester.toHostSpecs(nodes)); transaction.commit(); assertEquals(3, nodeRepository.getNodes(NodeType.tenant, Node.State.active).size()); @@ -133,14 +131,4 @@ public class FailedExpirerTest { return nodeRepository; } - - private Set<HostSpec> asHosts(List<Node> nodes) { - Set<HostSpec> hosts = new HashSet<>(nodes.size()); - for (Node node : nodes) - hosts.add(new HostSpec(node.hostname(), - node.allocation().isPresent() ? Optional.of(node.allocation().get().membership()) : - Optional.empty())); - return hosts; - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 9460d7f8c4a..bf954cf5feb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; @@ -19,10 +18,8 @@ import org.junit.Test; import java.time.Duration; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -39,12 +36,11 @@ public class InactiveAndFailedExpirerTest { public void ensure_inactive_and_failed_times_out() throws InterruptedException { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east"))); List<Node> nodes = tester.makeReadyNodes(2, "default"); - ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); // Allocate then deallocate 2 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); tester.prepare(applicationId, cluster, Capacity.fromNodeCount(2), 1); - tester.activate(applicationId, asHosts(nodes)); + tester.activate(applicationId, ProvisioningTester.toHostSpecs(nodes)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); tester.deactivate(applicationId); List<Node> inactiveNodes = tester.getNodes(applicationId, Node.State.inactive).asList(); @@ -81,7 +77,7 @@ public class InactiveAndFailedExpirerTest { // Allocate and deallocate a single node ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Optional.empty()); tester.prepare(applicationId, cluster, Capacity.fromNodeCount(1), 1); - tester.activate(applicationId, asHosts(nodes)); + tester.activate(applicationId, ProvisioningTester.toHostSpecs(nodes)); assertEquals(1, tester.getNodes(applicationId, Node.State.active).size()); tester.deactivate(applicationId); List<Node> inactiveNodes = tester.getNodes(applicationId, Node.State.inactive).asList(); @@ -100,14 +96,4 @@ public class InactiveAndFailedExpirerTest { // Reboot generation is increased assertEquals(wantedRebootGeneration + 1, dirty.get(0).status().reboot().wanted()); } - - private Set<HostSpec> asHosts(List<Node> nodes) { - Set<HostSpec> hosts = new HashSet<>(nodes.size()); - for (Node node : nodes) - hosts.add(new HostSpec(node.hostname(), - node.allocation().isPresent() ? Optional.of(node.allocation().get().membership()) : - Optional.empty())); - return hosts; - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java new file mode 100644 index 00000000000..8be57afedde --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester.createConfig; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class AclProvisioningTest { + + private MockCurator curator; + private ProvisioningTester tester; + + @Before + public void before() { + this.curator = new MockCurator(); + this.tester = new ProvisioningTester(Zone.defaultZone(), createConfig(), curator); + } + + @Test + public void trusted_nodes_for_allocated_node() { + String connectionSpec = "cfg1:1234,cfg2:1234,cfg3:1234"; + curator.setConnectionSpec(connectionSpec); + List<String> configServers = toHostNames(connectionSpec); + + // Populate repo + tester.makeReadyNodes(10, "default"); + List<Node> proxyNodes = tester.makeReadyNodes(3, "default", NodeType.proxy); + + ApplicationId applicationId = tester.makeApplicationId(); + + // Allocate 2 nodes + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), + Optional.empty()); + List<HostSpec> prepared = tester.prepare(applicationId, cluster, Capacity.fromNodeCount(2), 1); + tester.activate(applicationId, new HashSet<>(prepared)); + List<Node> activeNodes = tester.getNodes(applicationId, Node.State.active).asList(); + assertEquals(2, activeNodes.size()); + + // Get trusted nodes for the first active node + Node node = activeNodes.get(0); + List<Node> trustedNodes = tester.nodeRepository().getTrustedNodes(node); + assertEquals(activeNodes.size() + proxyNodes.size() + configServers.size(), trustedNodes.size()); + + // Trusted nodes contains active nodes in same application, proxy nodes and config servers + List<String> expected = flatten(Arrays.asList(toHostNames(activeNodes), toHostNames(proxyNodes), configServers)); + assertContainsOnly(toHostNames(trustedNodes), expected); + } + + @Test + public void trusted_nodes_for_unallocated_node() { + String connectionSpec = "cfg1:1234,cfg2:1234,cfg3:1234"; + curator.setConnectionSpec(connectionSpec); + List<String> configServers = toHostNames(connectionSpec); + + // Populate repo + List<Node> readyNodes = tester.makeReadyNodes(10, "default"); + List<Node> proxyNodes = tester.makeReadyNodes(3, "default", NodeType.proxy); + + // Get trusted nodes for the first ready node + Node node = readyNodes.get(0); + List<Node> trustedNodes = tester.nodeRepository().getTrustedNodes(node); + assertEquals(proxyNodes.size() + configServers.size(), trustedNodes.size()); + + // Trusted nodes contains only proxy nodes and config servers + assertContainsOnly(toHostNames(trustedNodes), flatten(Arrays.asList(toHostNames(proxyNodes), configServers))); + } + + private static <T> void assertContainsOnly(Collection<T> a, Collection<T> b) { + assertTrue(a.containsAll(b) && b.containsAll(a)); + } + + private static <T> List<T> flatten(List<List<T>> lists) { + return lists.stream().flatMap(Collection::stream).collect(Collectors.toList()); + } + + private static List<String> toHostNames(String connectionSpec) { + return Arrays.stream(connectionSpec.split(",")) + .map(hostPort -> hostPort.split(":")[0]) + .collect(Collectors.toList()); + } + + private static List<String> toHostNames(List<Node> node) { + return node.stream().map(Node::hostname).collect(Collectors.toList()); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 4352a64a3f9..36375f650b4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Flavor; import com.yahoo.vespa.hosted.provision.node.NodeFlavors; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; @@ -50,45 +51,39 @@ import static org.junit.Assert.assertTrue; */ public class ProvisioningTester implements AutoCloseable { - private Curator curator = new MockCurator(); - private NodeFlavors nodeFlavors; - private ManualClock clock; - private NodeRepository nodeRepository; - private NodeRepositoryProvisioner provisioner; - private CapacityPolicies capacityPolicies; - private ProvisionLogger provisionLogger; + private final Curator curator; + private final NodeFlavors nodeFlavors; + private final ManualClock clock; + private final NodeRepository nodeRepository; + private final NodeRepositoryProvisioner provisioner; + private final CapacityPolicies capacityPolicies; + private final ProvisionLogger provisionLogger; public ProvisioningTester(Zone zone) { - try { - nodeFlavors = new NodeFlavors(createConfig()); - clock = new ManualClock(); - nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, - new MockNameResolver().mockAnyLookup()); - provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock); - capacityPolicies = new CapacityPolicies(zone, nodeFlavors); - provisionLogger = new NullProvisionLogger(); - } - catch (Exception e) { - throw new RuntimeException(e); - } + this(zone, createConfig()); } public ProvisioningTester(Zone zone, NodeRepositoryConfig config) { + this(zone, config, new MockCurator()); + } + + public ProvisioningTester(Zone zone, NodeRepositoryConfig config, Curator curator) { try { - nodeFlavors = new NodeFlavors(config); - clock = new ManualClock(); - nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, + this.nodeFlavors = new NodeFlavors(config); + this.clock = new ManualClock(); + this.curator = curator; + this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup()); - provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock); - capacityPolicies = new CapacityPolicies(zone, nodeFlavors); - provisionLogger = new NullProvisionLogger(); + this.provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone, clock); + this.capacityPolicies = new CapacityPolicies(zone, nodeFlavors); + this.provisionLogger = new NullProvisionLogger(); } catch (Exception e) { throw new RuntimeException(e); } } - private NodeRepositoryConfig createConfig() { + public static NodeRepositoryConfig createConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3); b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2); @@ -266,6 +261,12 @@ public class ProvisioningTester implements AutoCloseable { return nodeRepository.getNode(hostname).map(Node::flavor).orElseThrow(() -> new RuntimeException("No flavor for host " + hostname)); } + public static Set<HostSpec> toHostSpecs(List<Node> nodes) { + return nodes.stream() + .map(node -> new HostSpec(node.hostname(), node.allocation().map(Allocation::membership))) + .collect(Collectors.toSet()); + } + private static class NullProvisionLogger implements ProvisionLogger { @Override diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 97f659a33fd..039f1646c38 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -15,6 +15,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -210,6 +211,23 @@ public class RestApiTest { } @Test + public void acl_request() throws Exception { + String hostName = "foo.yahoo.com"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson(hostName, "default") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + Pattern responsePattern = Pattern.compile("\\{\"hostname\":\"foo.yahoo.com\",\"ipAddress\":\".+?\"," + + "\"trustedNodes\":\\[" + + "\\{\"hostname\":\"cfg1\",\"ipAddress\":\".+?\"}," + + "\\{\"hostname\":\"cfg2\",\"ipAddress\":\".+?\"}," + + "\\{\"hostname\":\"cfg3\",\"ipAddress\":\".+?\"}" + + "]}"); + assertResponseMatches(new Request("http://localhost:8080/nodes/v2/acl/" + hostName), responsePattern); + } + + @Test public void test_invalid_requests() throws Exception { // Attempt to fail and ready an allocated node without going through dirty assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host1.yahoo.com", @@ -342,6 +360,11 @@ public class RestApiTest { container.handleRequest(request).getBodyAsString().contains(responseSnippet)); } + private void assertResponseMatches(Request request, Pattern pattern) throws IOException { + assertTrue("Response matches " + pattern.toString(), + pattern.matcher(container.handleRequest(request).getBodyAsString()).matches()); + } + private void assertFile(Request request, String responseFile) throws IOException { String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); expectedResponse = include(expectedResponse); diff --git a/persistence/src/tests/CMakeLists.txt b/persistence/src/tests/CMakeLists.txt index df9dd6cd1f4..8ff63707b2e 100644 --- a/persistence/src/tests/CMakeLists.txt +++ b/persistence/src/tests/CMakeLists.txt @@ -7,8 +7,9 @@ vespa_add_executable(persistence_testrunner_app TEST persistence_testspi ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME persistence_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:persistence_testrunner_app> + DEPENDS persistence_testrunner_app ) diff --git a/searchlib/src/tests/attribute/multi_value_mapping2/multi_value_mapping2_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping2/multi_value_mapping2_test.cpp index 4934bc8ea29..10398c18f68 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping2/multi_value_mapping2_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping2/multi_value_mapping2_test.cpp @@ -5,6 +5,7 @@ LOG_SETUP("multivaluemapping2_test"); #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/attribute/multi_value_mapping2.h> #include <vespa/searchlib/attribute/multi_value_mapping2.hpp> +#include <vespa/searchlib/attribute/not_implemented_attribute.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -15,16 +16,52 @@ assertArray(const std::vector<EntryT> &exp, vespalib::ConstArrayRef<EntryT> valu EXPECT_EQUAL(exp, std::vector<EntryT>(values.cbegin(), values.cend())); } +template <class MvMapping> +class MyAttribute : public search::NotImplementedAttribute +{ + using MultiValueType = typename MvMapping::MultiValueType; + using ConstArrayRef = vespalib::ConstArrayRef<MultiValueType>; + MvMapping &_mvMapping; + virtual void onCommit() { } + virtual void onUpdateStat() { } + virtual void onShrinkLidSpace() { + uint32_t committedDocIdLimit = getCommittedDocIdLimit(); + _mvMapping.shrink(committedDocIdLimit); + setNumDocs(committedDocIdLimit); + } + +public: + MyAttribute(MvMapping &mvMapping) + : NotImplementedAttribute("test", AttributeVector::Config()), + _mvMapping(mvMapping) + { + } + virtual bool addDoc(DocId &doc) { + _mvMapping.addDoc(doc); + incNumDocs(); + updateUncommittedDocIdLimit(doc); + return false; + } + virtual uint32_t clearDoc(uint32_t docId) { + assert(docId < _mvMapping.size()); + _mvMapping.set(docId, ConstArrayRef()); + return 1u; + } +}; + template <typename EntryT> class Fixture { - search::attribute::MultiValueMapping2<EntryT> _mvMapping; + using MvMapping = search::attribute::MultiValueMapping2<EntryT>; + MvMapping _mvMapping; + MyAttribute<MvMapping> _attr; using generation_t = vespalib::GenerationHandler::generation_t; public: using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; Fixture(uint32_t maxSmallArraySize) - : _mvMapping(maxSmallArraySize) + : _mvMapping(maxSmallArraySize), + _attr(_mvMapping) { } ~Fixture() { } @@ -38,6 +75,24 @@ public: } void transferHoldLists(generation_t generation) { _mvMapping.transferHoldLists(generation); } void trimHoldLists(generation_t firstUsed) { _mvMapping.trimHoldLists(firstUsed); } + void addDocs(uint32_t numDocs) { + for (uint32_t i = 0; i < numDocs; ++i) { + uint32_t doc = 0; + _attr.addDoc(doc); + } + _attr.commit(); + _attr.incGeneration(); + } + uint32_t size() const { return _mvMapping.size(); } + void shrink(uint32_t docIdLimit) { + _attr.setCommittedDocIdLimit(docIdLimit); + _attr.commit(); + _attr.incGeneration(); + _attr.shrinkLidSpace(); + } + void clearDocs(uint32_t lidLow, uint32_t lidLimit) { + _mvMapping.clearDocs(lidLow, lidLimit, _attr); + } }; TEST_F("Test that set and get works", Fixture<int>(3)) @@ -69,4 +124,35 @@ TEST_F("Test that old value is not overwritten while held", Fixture<int>(3)) TEST_DO(assertArray({0}, old3)); } +TEST_F("Test that addDoc works", Fixture<int>(3)) +{ + EXPECT_EQUAL(0, f.size()); + f.addDocs(10); + EXPECT_EQUAL(10u, f.size()); +} + +TEST_F("Test that shrink works", Fixture<int>(3)) +{ + f.addDocs(10); + EXPECT_EQUAL(10u, f.size()); + f.shrink(5); + EXPECT_EQUAL(5u, f.size()); +} + +TEST_F("Test that clearDocs works", Fixture<int>(3)) +{ + f.addDocs(10); + f.set(1, {}); + f.set(2, {4, 7}); + f.set(3, {5}); + f.set(4, {10, 14, 17, 16}); + f.set(5, {3}); + f.clearDocs(3, 5); + TEST_DO(f.assertGet(1, {})); + TEST_DO(f.assertGet(2, {4, 7})); + TEST_DO(f.assertGet(3, {})); + TEST_DO(f.assertGet(4, {})); + TEST_DO(f.assertGet(5, {3})); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/multivaluemapping/multivaluemapping_test.cpp b/searchlib/src/tests/attribute/multivaluemapping/multivaluemapping_test.cpp index 48768da32c5..ace75212300 100644 --- a/searchlib/src/tests/attribute/multivaluemapping/multivaluemapping_test.cpp +++ b/searchlib/src/tests/attribute/multivaluemapping/multivaluemapping_test.cpp @@ -16,7 +16,45 @@ uint32_t dummyCommittedDocIdLimit = std::numeric_limits<uint32_t>::max(); } -typedef MultiValueMappingT<uint32_t> MvMapping; +class MvMapping : public MultiValueMappingT<uint32_t> +{ + using ArrayRef = vespalib::ConstArrayRef<uint32_t>; +public: + using MultiValueMappingT<uint32_t>::MultiValueMappingT; + using MultiValueMappingT<uint32_t>::get; + + uint32_t getValueCount(uint32_t key) { + ArrayRef values = get(key); + return values.size(); + } + uint32_t get(uint32_t key, const uint32_t *&handle) { + ArrayRef values = get(key); + handle = &values[0]; + return values.size(); + } + uint32_t get(uint32_t key, uint32_t *buffer, uint32_t bufferSize) + { + ArrayRef values = get(key); + uint32_t valueCount = values.size(); + for (uint32_t i = 0, m(std::min(valueCount,bufferSize)); i < m; ++i) { + buffer[i] = values[i]; + } + return valueCount; + } + uint32_t get(uint32_t key, std::vector<uint32_t> &buffer) { + return get(key, &buffer[0], buffer.size()); + } + bool get(uint32_t key, uint32_t index, uint32_t &value) const { + ArrayRef values = get(key); + if (values.size() <= index) { + return false; + } else { + value = values[0]; + return true; + } + } +}; + typedef MvMapping::Index Index; typedef multivalue::Index64 Index64; typedef multivalue::Index32 Index32; @@ -205,7 +243,7 @@ MultiValueMappingTest::testSimpleSetAndGet() // add more keys for (uint32_t i = 0; i < 5; ++i) { uint32_t key; - mvm.addKey(key); + mvm.addDoc(key); EXPECT_TRUE(key == 10 + i); EXPECT_TRUE(mvm.getNumKeys() == 11 + i); } @@ -635,14 +673,14 @@ MultiValueMappingTest::testShrink() MvMapping mvm(committedDocIdLimit); for (uint32_t i = 0; i < 10; ++i) { uint32_t k; - mvm.addKey(k); + mvm.addDoc(k); EXPECT_EQUAL(i, k); } mvm.transferHoldLists(0); mvm.trimHoldLists(1); uint32_t shrinkTarget = 4; committedDocIdLimit = shrinkTarget; - mvm.shrinkKeys(shrinkTarget); + mvm.shrink(shrinkTarget); mvm.transferHoldLists(1); mvm.trimHoldLists(2); EXPECT_EQUAL(shrinkTarget, mvm.getNumKeys()); diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index a34f09dd87a..fa411a388d1 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -53,6 +53,7 @@ vespa_add_library(searchlib_attribute OBJECT loadedstringvalue.cpp loadedvalue.cpp multi_value_mapping2.cpp + multi_value_mapping2_base.cpp multienumattribute.cpp multienumattributesaver.cpp multinumericattribute.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.cpp index 3f946c79979..f80aa301429 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include "multi_value_mapping2.h" #include "multi_value_mapping2.hpp" -#include <vespa/vespalib/stllike/string.h> #include "multivalue.h" #include "enumstorebase.h" - -LOG_SETUP(".searchlib.attribute.multivaluemapping2"); +#include "attributevector.h" using search::multivalue::Value; using search::multivalue::WeightedValue; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.h index a177656a66d..52c09c68b3d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.h @@ -2,38 +2,31 @@ #pragma once -#include <vespa/searchlib/datastore/entryref.h> -#include <vespa/searchlib/common/rcuvector.h> +#include "multi_value_mapping2_base.h" #include <vespa/searchlib/datastore/array_store.h> #include "address_space.h" namespace search { - -class AttributeVector; - namespace attribute { /** * Class for mapping from from document id to an array of values. */ template <typename EntryT, typename RefT = datastore::EntryRefT<17> > -class MultiValueMapping2 +class MultiValueMapping2 : public MultiValueMapping2Base { public: using MultiValueType = EntryT; private: - using EntryRef = datastore::EntryRef; - using IndexVector = RcuVectorBase<EntryRef>; using ArrayStore = datastore::ArrayStore<EntryT, RefT>; using generation_t = vespalib::GenerationHandler::generation_t; using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; ArrayStore _store; - IndexVector _indices; public: MultiValueMapping2(uint32_t maxSmallArraySize, const GrowStrategy &gs = GrowStrategy()); - ~MultiValueMapping2(); + virtual ~MultiValueMapping2(); ConstArrayRef get(uint32_t docId) const { return _store.get(_indices[docId]); } ConstArrayRef getDataForIdx(EntryRef idx) const { return _store.get(idx); } void set(uint32_t docId, ConstArrayRef values); @@ -52,38 +45,12 @@ public: // Following methods are not yet properly implemented. AddressSpace getAddressSpaceUsage() const { return AddressSpace(0, 0); } - MemoryUsage getMemoryUsage() const { return MemoryUsage(); } - size_t getTotalValueCnt() const { return 0; } - void clearDocs(uint32_t lidLow, uint32_t lidLimit, AttributeVector &v) { - (void) lidLow; - (void) lidLimit; - (void) v; - } - void shrinkKeys(uint32_t newSize) { (void) newSize; } - void addKey(uint32_t &docId) { - uint32_t oldVal = _indices.size(); - _indices.push_back(EntryRef()); - docId = oldVal; - } + virtual MemoryUsage getMemoryUsage() const override { return MemoryUsage(); } + virtual size_t getTotalValueCnt() const override { return 0; } // Mockups to temporarily silence code written for old multivalue mapping - class Histogram - { - private: - using HistogramM = std::vector<size_t>; - public: - using const_iterator = HistogramM::const_iterator; - Histogram() : _histogram(1) { } - size_t & operator [] (uint32_t) { return _histogram[0]; } - const_iterator begin() const { return _histogram.begin(); } - const_iterator end() const { return _histogram.end(); } - private: - HistogramM _histogram; - }; - Histogram getEmptyHistogram() const { return Histogram(); } bool enoughCapacity(const Histogram &) { return true; } void performCompaction(Histogram &) { } - static size_t maxValues() { return 0; } }; } // namespace search::attribute diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.hpp index 5518721193f..188d98a0c24 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.hpp @@ -9,11 +9,8 @@ namespace attribute { template <typename EntryT, typename RefT> MultiValueMapping2<EntryT,RefT>::MultiValueMapping2(uint32_t maxSmallArraySize, const GrowStrategy &gs) - : _store(maxSmallArraySize), - _indices(gs.getDocsInitialCapacity(), - gs.getDocsGrowPercent(), - gs.getDocsGrowDelta(), - _store.getGenerationHolder()) + : MultiValueMapping2Base(gs, _store.getGenerationHolder()), + _store(maxSmallArraySize) { } @@ -22,7 +19,6 @@ MultiValueMapping2<EntryT,RefT>::~MultiValueMapping2() { } - template <typename EntryT, typename RefT> void MultiValueMapping2<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values) diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2_base.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2_base.cpp new file mode 100644 index 00000000000..f2ccfc71457 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2_base.cpp @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "multi_value_mapping2_base.h" +#include "attributevector.h" + +namespace search { +namespace attribute { + +MultiValueMapping2Base::MultiValueMapping2Base(const GrowStrategy &gs, + vespalib::GenerationHolder &genHolder) + : _indices(gs, genHolder) +{ +} + +MultiValueMapping2Base::~MultiValueMapping2Base() +{ +} + +MultiValueMapping2Base::RefCopyVector +MultiValueMapping2Base::getRefCopy(uint32_t size) const { + assert(size <= _indices.size()); + return RefCopyVector(&_indices[0], &_indices[0] + size); +} + +void +MultiValueMapping2Base::addDoc(uint32_t & docId) +{ + uint32_t retval = _indices.size(); + _indices.push_back(EntryRef()); + docId = retval; +} + +void +MultiValueMapping2Base::shrink(uint32_t docIdLimit) +{ + assert(docIdLimit < _indices.size()); + _indices.shrink(docIdLimit); +} + +void +MultiValueMapping2Base::clearDocs(uint32_t lidLow, uint32_t lidLimit, AttributeVector &v) +{ + assert(lidLow <= lidLimit); + assert(lidLimit <= v.getNumDocs()); + assert(lidLimit <= _indices.size()); + for (uint32_t lid = lidLow; lid < lidLimit; ++lid) { + if (_indices[lid].valid()) { + v.clearDoc(lid); + } + } +} + +} // namespace search::attribute +} // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2_base.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2_base.h new file mode 100644 index 00000000000..22c656d519d --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2_base.h @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/datastore/entryref.h> +#include <vespa/searchlib/common/rcuvector.h> + +namespace search { + +class AttributeVector; + +namespace attribute { + +/** + * Base class for mapping from from document id to an array of values. + */ +class MultiValueMapping2Base +{ +public: + using EntryRef = datastore::EntryRef; + using RefVector = RcuVectorBase<EntryRef>; + +protected: + RefVector _indices; + + MultiValueMapping2Base(const GrowStrategy &gs, vespalib::GenerationHolder &genHolder); + virtual ~MultiValueMapping2Base(); + +public: + using RefCopyVector = vespalib::Array<EntryRef>; + + virtual MemoryUsage getMemoryUsage() const = 0; + virtual size_t getTotalValueCnt() const = 0; + RefCopyVector getRefCopy(uint32_t size) const; + + void addDoc(uint32_t &docId); + void shrink(uint32_t docidLimit); + void clearDocs(uint32_t lidLow, uint32_t lidLimit, AttributeVector &v); + uint32_t size() const { return _indices.size(); } + + // Mockups to temporarily silence code written for old multivalue mapping + class Histogram + { + private: + using HistogramM = std::vector<size_t>; + public: + using const_iterator = HistogramM::const_iterator; + Histogram() : _histogram(1) { } + size_t & operator [] (uint32_t) { return _histogram[0]; } + const_iterator begin() const { return _histogram.begin(); } + const_iterator end() const { return _histogram.end(); } + private: + HistogramM _histogram; + }; + Histogram getEmptyHistogram() const { return Histogram(); } + static size_t maxValues() { return 0; } +}; + +} // namespace search::attribute +} // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.h b/searchlib/src/vespa/searchlib/attribute/multienumattribute.h index 70f2c779702..9d2e7321131 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.h @@ -35,6 +35,7 @@ protected: typedef typename EnumStoreBase::EnumVector EnumVector; typedef typename MultiValueAttribute<B, M>::MultiValueType WeightedIndex; typedef typename MultiValueAttribute<B, M>::ValueVector WeightedIndexVector; + using WeightedIndexArrayRef = typename MultiValueAttribute<B, M>::MultiValueArrayRef; typedef typename MultiValueAttribute<B, M>::Histogram Histogram; typedef typename MultiValueAttribute<B, M>::DocumentValues DocIndices; typedef AttributeVector::ReaderBase ReaderBase; @@ -86,25 +87,24 @@ public: // Attribute read API //----------------------------------------------------------------------------------------------------------------- virtual EnumHandle getEnum(DocId doc) const { - if (this->getValueCount(doc) == 0) { + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + if (indices.size() == 0) { return std::numeric_limits<uint32_t>::max(); } else { - WeightedIndex idx; - this->_mvMapping.get(doc, 0, idx); - return idx.value().ref(); + return indices[0].value().ref(); } } virtual uint32_t get(DocId doc, EnumHandle * e, uint32_t sz) const { - const WeightedIndex * indices; - uint32_t valueCount = this->_mvMapping.get(doc, indices); + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + uint32_t valueCount = indices.size(); for (uint32_t i = 0, m = std::min(sz, valueCount); i < m; ++i) { e[i] = indices[i].value().ref(); } return valueCount; } virtual uint32_t get(DocId doc, WeightedEnum * e, uint32_t sz) const { - const WeightedIndex * indices; - uint32_t valueCount = this->_mvMapping.get(doc, indices); + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + uint32_t valueCount = indices.size(); for (uint32_t i = 0, m = std::min(sz, valueCount); i < m; ++i) { e[i] = WeightedEnum(indices[i].value().ref(), indices[i].weight()); } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h index 63cf52a42bd..8135576c8b0 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h @@ -39,6 +39,7 @@ private: typedef typename MultiValueAttribute<B, M>::Change Change; typedef typename MultiValueAttribute<B, M>::ValueType MValueType; // = B::BaseType typedef typename MultiValueAttribute<B, M>::MultiValueType MultiValueType; // = B::BaseType + using MultiValueArrayRef = typename MultiValueAttribute<B, M>::MultiValueArrayRef; virtual bool extractChangeData(const Change & c, MValueType & data) { data = static_cast<MValueType>(c._data.get()); @@ -57,7 +58,11 @@ private: protected: typedef typename B::generation_t generation_t; typedef MultiValueType WType; - uint32_t get(DocId doc, const WType * & values) const { return this->_mvMapping.get(doc, values); } + uint32_t get(DocId doc, const WType * & values) const { + MultiValueArrayRef array(this->_mvMapping.get(doc)); + values = &array[0]; + return array.size(); + } public: virtual uint32_t getRawValues(DocId doc, const WType * & values) const { return get(doc, values); } @@ -98,12 +103,10 @@ public: bool cmp(DocId doc, int32_t & weight) const { - const MultiValueType * buffer; - for (uint32_t i = 0, m = _toBeSearched._mvMapping.get(doc, buffer); - i < m; i++) { - T v(buffer[i].value()); - if (this->match(v)) { - weight = buffer[i].weight(); + MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); + for (const MultiValueType &mv : values) { + if (this->match(mv.value())) { + weight = mv.weight(); return true; } } @@ -113,11 +116,9 @@ public: bool cmp(DocId doc) const { - const MultiValueType * buffer; - for (uint32_t i = 0, m = _toBeSearched._mvMapping.get(doc, buffer); - i < m; i++) { - T v(buffer[i].value()); - if (this->match(v)) { + MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); + for (const MultiValueType &mv : values) { + if (this->match(mv.value())) { return true; } } @@ -179,11 +180,9 @@ public: cmp(DocId doc, int32_t & weight) const { uint32_t hitCount = 0; - const MultiValueType * buffer; - for (uint32_t i = 0, m = _toBeSearched._mvMapping.get(doc, buffer); - i < m; i++) { - T v = buffer[i].value(); - if (this->match(v)) { + MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); + for (const MultiValueType &mv : values) { + if (this->match(mv.value())) { hitCount++; } } @@ -195,11 +194,9 @@ public: bool cmp(DocId doc) const { - const MultiValueType * buffer; - for (uint32_t i = 0, m = _toBeSearched._mvMapping.get(doc, buffer); - i < m; i++) { - T v = buffer[i].value(); - if (this->match(v)) { + MultiValueArrayRef values(_toBeSearched._mvMapping.get(doc)); + for (const MultiValueType &mv : values) { + if (this->match(mv.value())) { return true; } } @@ -255,19 +252,16 @@ public: // new read api //------------------------------------------------------------------------- virtual T get(DocId doc) const { - MultiValueType value; - this->_mvMapping.get(doc, 0, value); - return value; + MultiValueArrayRef values(this->_mvMapping.get(doc)); + return ((values.size() > 0) ? values[0].value() : T()); } virtual largeint_t getInt(DocId doc) const { - MultiValueType value; - this->_mvMapping.get(doc, 0, value); - return static_cast<largeint_t>(value.value()); + MultiValueArrayRef values(this->_mvMapping.get(doc)); + return static_cast<largeint_t>((values.size() > 0) ? values[0].value() : T()); } virtual double getFloat(DocId doc) const { - MultiValueType value; - this->_mvMapping.get(doc, 0, value); - return static_cast<double>(value.value()); + MultiValueArrayRef values(this->_mvMapping.get(doc)); + return static_cast<double>((values.size() > 0) ? values[0].value() : T()); } virtual EnumHandle getEnum(DocId doc) const { (void) doc; @@ -284,8 +278,8 @@ public: } template <typename BufferType> uint32_t getHelper(DocId doc, BufferType * buffer, uint32_t sz) const { - const MultiValueType * handle; - uint32_t ret = this->_mvMapping.get(doc, handle); + MultiValueArrayRef handle(this->_mvMapping.get(doc)); + uint32_t ret = handle.size(); for(size_t i(0), m(std::min(sz, ret)); i < m; i++) { buffer[i] = static_cast<BufferType>(handle[i].value()); } @@ -299,7 +293,8 @@ public: } template <typename E> uint32_t getEnumHelper(DocId doc, E * e, uint32_t sz) const { - uint32_t available = getValueCount(doc); + MultiValueArrayRef values(this->_mvMapping.get(doc)); + uint32_t available = values.size(); uint32_t num2Read = std::min(available, sz); for (uint32_t i = 0; i < num2Read; ++i) { e[i] = E(std::numeric_limits<uint32_t>::max()); // does not have enum @@ -317,8 +312,8 @@ public: } template <typename WeightedType, typename ValueType> uint32_t getWeightedHelper(DocId doc, WeightedType * buffer, uint32_t sz) const { - const MultiValueType * handle; - uint32_t ret = this->_mvMapping.get(doc, handle); + MultiValueArrayRef handle(this->_mvMapping.get(doc)); + uint32_t ret = handle.size(); for(size_t i(0), m(std::min(sz, ret)); i < m; i++) { buffer[i] = WeightedType(static_cast<ValueType>(handle[i].value()), handle[i].weight()); diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index a593472df9b..5983d0320ed 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -39,7 +39,8 @@ uint32_t MultiValueNumericAttribute<B, M>::getValueCount(DocId doc) const if (doc >= B::getNumDocs()) { return 0; } - return this->_mvMapping.getValueCount(doc); + MultiValueArrayRef values(this->_mvMapping.get(doc)); + return values.size(); } template <typename B, typename M> diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h index fae23b72dba..8cd24c2cc38 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h @@ -36,6 +36,7 @@ protected: typedef typename B::BaseClass::WeightedEnum WeightedEnum; typedef typename MultiValueEnumAttribute<B, M>::MultiValueType WeightedIndex; + using WeightedIndexArrayRef = typename MultiValueEnumAttribute<B, M>::MultiValueArrayRef; typedef attribute::LoadedEnumAttribute LoadedEnumAttribute; typedef attribute::LoadedEnumAttributeVector LoadedEnumAttributeVector; typedef EnumStoreBase::IndexVector EnumIndexVector; @@ -76,12 +77,11 @@ protected: bool cmp(DocId doc, int32_t & weight) const { - const WeightedIndex * indices; - uint32_t valueCount = _toBeSearched._mvMapping.get(doc, indices); - for (uint32_t i = 0; i < valueCount; ++i) { - T v = _toBeSearched._enumStore.getValue(indices[i].value()); + WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); + for (const WeightedIndex &wi : indices) { + T v = _toBeSearched._enumStore.getValue(wi.value()); if (this->match(v)) { - weight = indices[i].weight(); + weight = wi.weight(); return true; } } @@ -91,10 +91,9 @@ protected: bool cmp(DocId doc) const { - const WeightedIndex * indices; - uint32_t valueCount = _toBeSearched._mvMapping.get(doc, indices); - for (uint32_t i = 0; i < valueCount; ++i) { - T v = _toBeSearched._enumStore.getValue(indices[i].value()); + WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); + for (const WeightedIndex &wi : indices) { + T v = _toBeSearched._enumStore.getValue(wi.value()); if (this->match(v)) { return true; } @@ -163,10 +162,9 @@ protected: cmp(DocId doc, int32_t & weight) const { uint32_t hitCount = 0; - const WeightedIndex * indices; - uint32_t valueCount = _toBeSearched._mvMapping.get(doc, indices); - for (uint32_t i = 0; i < valueCount; ++i) { - T v = _toBeSearched._enumStore.getValue(indices[i].value()); + WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); + for (const WeightedIndex &wi : indices) { + T v = _toBeSearched._enumStore.getValue(wi.value()); if (this->match(v)) { hitCount++; } @@ -179,10 +177,9 @@ protected: bool cmp(DocId doc) const { - const WeightedIndex * indices; - uint32_t valueCount = _toBeSearched._mvMapping.get(doc, indices); - for (uint32_t i = 0; i < valueCount; ++i) { - T v = _toBeSearched._enumStore.getValue(indices[i].value()); + WeightedIndexArrayRef indices(_toBeSearched._mvMapping.get(doc)); + for (const WeightedIndex &wi : indices) { + T v = _toBeSearched._enumStore.getValue(wi.value()); if (this->match(v)) { return true; } @@ -227,12 +224,11 @@ public: // Attribute read API //------------------------------------------------------------------------- virtual T get(DocId doc) const { - if (this->getValueCount(doc) == 0) { + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + if (indices.size() == 0) { return T(); } else { - WeightedIndex idx; - this->_mvMapping.get(doc, 0, idx); - return this->_enumStore.getValue(idx.value()); + return this->_enumStore.getValue(indices[0].value()); } } virtual largeint_t getInt(DocId doc) const { @@ -244,8 +240,8 @@ public: template <typename BufferType> uint32_t getHelper(DocId doc, BufferType * buffer, uint32_t sz) const { - const WeightedIndex * indices; - uint32_t valueCount = this->_mvMapping.get(doc, indices); + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + uint32_t valueCount = indices.size(); for(uint32_t i = 0, m = std::min(sz, valueCount); i < m; i++) { buffer[i] = static_cast<BufferType>(this->_enumStore.getValue(indices[i].value())); } @@ -263,8 +259,8 @@ public: template <typename WeightedType, typename ValueType> uint32_t getWeightedHelper(DocId doc, WeightedType * buffer, uint32_t sz) const { - const WeightedIndex * indices; - uint32_t valueCount = this->_mvMapping.get(doc, indices); + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + uint32_t valueCount = indices.size(); for (uint32_t i = 0, m = std::min(sz, valueCount); i < m; ++i) { buffer[i] = WeightedType(static_cast<ValueType>(this->_enumStore.getValue(indices[i].value())), indices[i].weight()); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index 2f740cd6b30..ac250d62f60 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -31,6 +31,7 @@ protected: typedef typename MultiValueAttribute<B, M>::ValueType EnumIndex; typedef typename MultiValueAttribute<B, M>::MultiValueMapping MultiValueMapping; typedef typename MultiValueAttribute<B, M>::ValueVector WeightedIndexVector; + using WeightedIndexArrayRef = typename MultiValueAttribute<B, M>::MultiValueArrayRef; typedef typename MultiValueAttribute<B, M>::DocumentValues DocIndices; typedef StringAttribute::DocId DocId; @@ -65,18 +66,17 @@ public: // new read api //------------------------------------------------------------------------- virtual const char * get(DocId doc) const { - if (this->getValueCount(doc) == 0) { + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + if (indices.size() == 0) { return NULL; } else { - WeightedIndex idx; - this->_mvMapping.get(doc, 0, idx); - return this->_enumStore.getValue(idx.value()); + return this->_enumStore.getValue(indices[0].value()); } } template <typename BufferType> uint32_t getHelper(DocId doc, BufferType * buffer, uint32_t sz) const { - const WeightedIndex * indices; - uint32_t valueCount = this->_mvMapping.get(doc, indices); + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + uint32_t valueCount = indices.size(); for(uint32_t i = 0, m = std::min(sz, valueCount); i < m; i++) { buffer[i] = this->_enumStore.getValue(indices[i].value()); } @@ -92,8 +92,8 @@ public: /// Weighted interface template <typename WeightedType> uint32_t getWeightedHelper(DocId doc, WeightedType * buffer, uint32_t sz) const { - const WeightedIndex * indices; - uint32_t valueCount = this->_mvMapping.get(doc, indices); + WeightedIndexArrayRef indices(this->_mvMapping.get(doc)); + uint32_t valueCount = indices.size(); for (uint32_t i = 0, m = std::min(sz, valueCount); i < m; ++i) { buffer[i] = WeightedType(this->_enumStore.getValue(indices[i].value()), indices[i].weight()); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index e791adb3231..0e0c51cb16c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -89,11 +89,10 @@ template <typename Collector> bool MultiValueStringAttributeT<B, M>::StringImplSearchContext::collectWeight(DocId doc, int32_t & weight, Collector & collector) const { - const WeightedIndex * indices; - uint32_t valueCount = myAttribute()._mvMapping.get(doc, indices); + WeightedIndexArrayRef indices(myAttribute()._mvMapping.get(doc)); EnumAccessor<typename B::EnumStore> accessor(myAttribute()._enumStore); - collectMatches(indices, valueCount, accessor, collector); + collectMatches(indices, accessor, collector); weight = collector.getWeight(); return collector.hasMatch(); } @@ -103,11 +102,9 @@ bool MultiValueStringAttributeT<B, M>::StringImplSearchContext::onCmp(DocId doc) const { const MultiValueStringAttributeT<B, M> & attr(static_cast< const MultiValueStringAttributeT<B, M> & > (attribute())); - const WeightedIndex * indices; - uint32_t valueCount = attr._mvMapping.get(doc, indices); - - for (uint32_t i(0); (i < valueCount); i++) { - if (isMatch(attr._enumStore.getValue(indices[i].value()))) { + WeightedIndexArrayRef indices(attr._mvMapping.get(doc)); + for (const WeightedIndex &wi : indices) { + if (isMatch(attr._enumStore.getValue(wi.value()))) { return true; } } diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index 4d94e74d37e..639602a698c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -26,6 +26,7 @@ protected: typedef typename MultiValueMappingBaseBase::Histogram Histogram; typedef typename MultiValueType::ValueType ValueType; typedef std::vector<MultiValueType> ValueVector; + using MultiValueArrayRef = vespalib::ConstArrayRef<MultiValueType>; typedef typename ValueVector::iterator ValueVectorIterator; typedef std::vector<std::pair<DocId, ValueVector> > DocumentValues; diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 759364b4f93..f8ad27c95a8 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -23,9 +23,8 @@ MultiValueAttribute<B, M>::~MultiValueAttribute() template <typename B, typename M> int32_t MultiValueAttribute<B, M>::getWeight(DocId doc, uint32_t idx) const { - MultiValueType value; - this->_mvMapping.get(doc, idx, value); - return (value.weight()); + MultiValueArrayRef values(this->_mvMapping.get(doc)); + return ((idx < values.size()) ? values[idx].weight() : 1); } @@ -39,8 +38,8 @@ MultiValueAttribute<B, M>::applyAttributeChanges(DocumentValues & docValues) for (ChangeVectorIterator current(this->_changes.begin()), end(this->_changes.end()); (current != end); ) { DocId doc = current->_doc; - ValueVector newValues(_mvMapping.getValueCount(doc)); - _mvMapping.get(doc, newValues); + MultiValueArrayRef oldValues(_mvMapping.get(doc)); + ValueVector newValues(oldValues.cbegin(), oldValues.cend()); // find last clear doc ChangeVectorIterator lastClearDoc = end; @@ -151,7 +150,7 @@ bool MultiValueAttribute<B, M>::addDoc(DocId & doc) { bool incGen = this->_mvMapping.isFull(); - this->_mvMapping.addKey(doc); + this->_mvMapping.addDoc(doc); this->incNumDocs(); this->updateUncommittedDocIdLimit(doc); incGen |= onAddDoc(doc); @@ -169,7 +168,8 @@ MultiValueAttribute<B, M>::getValueCount(DocId doc) const if (doc >= this->getNumDocs()) { return 0; } - return this->_mvMapping.getValueCount(doc); + MultiValueArrayRef values(this->_mvMapping.get(doc)); + return values.size(); } @@ -194,7 +194,7 @@ void MultiValueAttribute<B, M>::onShrinkLidSpace() { uint32_t committedDocIdLimit = this->getCommittedDocIdLimit(); - _mvMapping.shrinkKeys(committedDocIdLimit); + _mvMapping.shrink(committedDocIdLimit); this->setNumDocs(committedDocIdLimit); } diff --git a/searchlib/src/vespa/searchlib/attribute/multivaluemapping.cpp b/searchlib/src/vespa/searchlib/attribute/multivaluemapping.cpp index 47367b974bf..df61ccf2a22 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivaluemapping.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multivaluemapping.cpp @@ -230,21 +230,21 @@ MultiValueMappingBase<I>::reset(uint32_t numKeys) template <typename I> void -MultiValueMappingBase<I>::addKey(uint32_t & key) +MultiValueMappingBase<I>::addDoc(uint32_t & docId) { uint32_t retval = _indices.size(); _indices.push_back(Index()); - key = retval; + docId = retval; } template <typename I> void -MultiValueMappingBase<I>::shrinkKeys(uint32_t newSize) +MultiValueMappingBase<I>::shrink(uint32_t docIdLimit) { - assert(newSize >= _committedDocIdLimit); - assert(newSize < _indices.size()); - _indices.shrink(newSize); + assert(docIdLimit >= _committedDocIdLimit); + assert(docIdLimit < _indices.size()); + _indices.shrink(docIdLimit); } diff --git a/searchlib/src/vespa/searchlib/attribute/multivaluemapping.h b/searchlib/src/vespa/searchlib/attribute/multivaluemapping.h index fb2c39b89fe..f7af19eb530 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivaluemapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multivaluemapping.h @@ -209,8 +209,8 @@ public: bool hasKey(uint32_t key) const { return key < _indices.size(); } bool isFull() const { return _indices.isFull(); } - void addKey(uint32_t & key); - void shrinkKeys(uint32_t newSize); + void addDoc(uint32_t & docId); + void shrink(uint32_t docIdLimit); void clearDocs(uint32_t lidLow, uint32_t lidLimit, AttributeVector &v); void holdElem(Index idx, size_t size); virtual void doneHoldElem(Index idx) = 0; @@ -361,29 +361,6 @@ public: ~MultiValueMappingT(); void reset(uint32_t numKeys, size_t initSize = 0); void reset(uint32_t numKeys, const Histogram & initCapacity); - uint32_t get(uint32_t key, std::vector<T> & buffer) const; - template <typename BufferType> - uint32_t get(uint32_t key, BufferType * buffer, uint32_t sz) const; - bool get(uint32_t key, uint32_t index, T & value) const; - uint32_t getDataForIdx(Index idx, const T * & handle) const { - if (__builtin_expect(idx.values() < Index::maxValues(), true)) { - // We do not need to specialcase 0 as _singleVectors will refer to valid stuff - // and handle SHALL not be used as the number of values returned shall be obeyed. - const SingleVector & vec = _singleVectors[idx.vectorIdx()]; - handle = &vec[idx.offset() * idx.values()]; - __builtin_prefetch(handle, 0, 0); - return idx.values(); - } else { - const VectorBase & vec = - _vectorVectors[idx.alternative()][idx.offset()]; - handle = &vec[0]; - return vec.size(); - } - } - uint32_t get(uint32_t key, const T * & handle) const { - return getDataForIdx(this->_indices[key], handle); - } - inline uint32_t getValueCount(uint32_t key) const; vespalib::ConstArrayRef<T> getDataForIdx(Index idx) const { if (__builtin_expect(idx.values() < Index::maxValues(), true)) { // We do not need to specialcase 0 as _singleVectors will refer to valid stuff @@ -805,82 +782,6 @@ MultiValueMappingT<T, I>::reset(uint32_t numKeys, template <typename T, typename I> -uint32_t -MultiValueMappingT<T, I>::get(uint32_t key, std::vector<T> & buffer) const -{ - return get(key, &buffer[0], buffer.size()); -} - -template <typename T, typename I> -template <typename BufferType> -uint32_t -MultiValueMappingT<T, I>::get(uint32_t key, - BufferType * buffer, - uint32_t sz) const -{ - Index idx = this->_indices[key]; - if (idx.values() < Index::maxValues()) { - uint32_t available = idx.values(); - uint32_t num2Read = std::min(available, sz); - const SingleVector & vec = _singleVectors[idx.vectorIdx()]; - for (uint64_t i = 0, j = idx.offset() * idx.values(); - i < num2Read && j < (idx.offset() + 1) * idx.values(); ++i, ++j) { - buffer[i] = static_cast<BufferType>(vec[j]); - } - return available; - } else { - const VectorBase & vec = - _vectorVectors[idx.alternative()][idx.offset()]; - uint32_t available = vec.size(); - uint32_t num2Read = std::min(available, sz); - for (uint32_t i = 0; i < num2Read; ++i) { - buffer[i] = static_cast<BufferType>(vec[i]); - } - return available; - } -} - -template <typename T, typename I> -bool -MultiValueMappingT<T, I>::get(uint32_t key, uint32_t index, T & value) const -{ - if (!this->hasReaderKey(key)) { - return false; - } - Index idx = this->_indices[key]; - if (idx.values() < Index::maxValues()) { - if (index >= idx.values()) { - return false; - } - uint64_t offset = idx.offset() * idx.values() + index; - value = _singleVectors[idx.vectorIdx()][offset]; - return true; - } else { - if (index >= _vectorVectors[idx.alternative()][idx.offset()].size()) { - return false; - } - value = _vectorVectors[idx.alternative()][idx.offset()][index]; - return true; - } - return false; -} - -template <typename T, typename I> -inline uint32_t -MultiValueMappingT<T, I>::getValueCount(uint32_t key) const -{ - if (!this->hasReaderKey(key)) { - return 0; - } - Index idx = this->_indices[key]; - if (idx.values() < Index::maxValues()) { - return idx.values(); - } else { - return _vectorVectors[idx.alternative()][idx.offset()].size(); - } -} - -template <typename T, typename I> void MultiValueMappingT<T, I>::set(uint32_t key, const std::vector<T> & values) { diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index a93b5904446..6024a68fa33 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -267,7 +267,7 @@ StringAttribute::StringSearchContext::onCmp(DocId docId, int32_t & weight) const CollectWeight collector; DirectAccessor accessor; - collectMatches(buffer, std::min(valueCount, _bufferLen), accessor, collector); + collectMatches(vespalib::ConstArrayRef<WeightedConstChar>(buffer, std::min(valueCount, _bufferLen)), accessor, collector); weight = collector.getWeight(); return collector.hasMatch(); } diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index f4e908634c9..71129d6d0b3 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -164,10 +164,10 @@ private: }; template<typename WeightedT, typename Accessor, typename Collector> - void collectMatches(const WeightedT * w, size_t sz, const Accessor & ac, Collector & collector) const { - for (uint32_t i(0); i < sz; i++) { - if (isMatch(ac.get(w[i].value()))) { - collector.addWeight(w[i].weight()); + void collectMatches(vespalib::ConstArrayRef<WeightedT> w, const Accessor & ac, Collector & collector) const { + for (const WeightedT &wRef : w) { + if (isMatch(ac.get(wRef.value()))) { + collector.addWeight(wRef.weight()); } } } diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt index bcceb12a935..f9b6575c4f7 100644 --- a/storage/src/tests/CMakeLists.txt +++ b/storage/src/tests/CMakeLists.txt @@ -17,8 +17,9 @@ vespa_add_executable(storage_testrunner_app TEST storage_teststatus ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME storage_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:storage_testrunner_app> + DEPENDS storage_testrunner_app ) diff --git a/storageapi/src/tests/CMakeLists.txt b/storageapi/src/tests/CMakeLists.txt index ad7117076fe..a7b0f5f5f8b 100644 --- a/storageapi/src/tests/CMakeLists.txt +++ b/storageapi/src/tests/CMakeLists.txt @@ -9,8 +9,9 @@ vespa_add_executable(storageapi_testrunner_app TEST storageapi ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME storageapi_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:storageapi_testrunner_app> + DEPENDS storageapi_testrunner_app ) diff --git a/storageframework/src/tests/CMakeLists.txt b/storageframework/src/tests/CMakeLists.txt index 9f39a171cba..883dd37d325 100644 --- a/storageframework/src/tests/CMakeLists.txt +++ b/storageframework/src/tests/CMakeLists.txt @@ -9,8 +9,9 @@ vespa_add_executable(storageframework_testrunner_app TEST storageframework_testthread ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME storageframework_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:storageframework_testrunner_app> + DEPENDS storageframework_testrunner_app ) diff --git a/storageserver/src/tests/CMakeLists.txt b/storageserver/src/tests/CMakeLists.txt index a0d00400f7c..142ab46acb6 100644 --- a/storageserver/src/tests/CMakeLists.txt +++ b/storageserver/src/tests/CMakeLists.txt @@ -11,8 +11,9 @@ vespa_add_executable(storageserver_testrunner_app TEST searchlib_searchlib_uca ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME storageserver_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:storageserver_testrunner_app> + DEPENDS storageserver_testrunner_app ) diff --git a/vdslib/src/tests/CMakeLists.txt b/vdslib/src/tests/CMakeLists.txt index a63560c0d77..c7777469ddd 100644 --- a/vdslib/src/tests/CMakeLists.txt +++ b/vdslib/src/tests/CMakeLists.txt @@ -9,8 +9,9 @@ vespa_add_executable(vdslib_testrunner_app TEST vdslib_testthread ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME vdslib_testrunner_app COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:vdslib_testrunner_app> + DEPENDS vdslib_testrunner_app ) diff --git a/vdstestlib/src/tests/cppunit/CMakeLists.txt b/vdstestlib/src/tests/cppunit/CMakeLists.txt index 062a021ac41..c53caaea385 100644 --- a/vdstestlib/src/tests/cppunit/CMakeLists.txt +++ b/vdstestlib/src/tests/cppunit/CMakeLists.txt @@ -7,9 +7,10 @@ vespa_add_executable(vdstestlib_testrunner_app TEST vdstestlib ) -# TODO: Test with a larget chunk size to parallelize test suite runs +# TODO: Test with a larger chunk size to parallelize test suite runs vespa_add_test( NAME vdstestlib_testrunner_app NO_VALGRIND COMMAND python ${PROJECT_SOURCE_DIR}/cppunit-parallelize.py --chunks 1 $<TARGET_FILE:vdstestlib_testrunner_app> + DEPENDS vdstestlib_testrunner_app ) |