diff options
45 files changed, 271 insertions, 386 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index e6d61e7c7a5..d6a34fdb834 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -36,9 +36,7 @@ get_env_var_with_optional_default() { fi } -# TODO: use maven-wrapper after changing mvn command for vespa in factory/build-java.sh -#readonly MAVEN_CMD=$(get_env_var_with_optional_default VESPA_MAVEN_COMMAND "$(pwd)/mvnw") -readonly MAVEN_CMD=$(get_env_var_with_optional_default VESPA_MAVEN_COMMAND mvn) +readonly MAVEN_CMD=$(get_env_var_with_optional_default VESPA_MAVEN_COMMAND "$(pwd)/mvnw") readonly MAVEN_EXTRA_OPTS=$(get_env_var_with_optional_default VESPA_MAVEN_EXTRA_OPTS) echo "Using maven command: ${MAVEN_CMD}" diff --git a/client/go/go.mod b/client/go/go.mod index 0d67283104f..d797017a810 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -17,7 +17,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/zalando/go-keyring v0.2.3 golang.org/x/net v0.14.0 - golang.org/x/sys v0.11.0 + golang.org/x/sys v0.12.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/client/go/go.sum b/client/go/go.sum index fbe0fa1207e..4bea3accfae 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -74,6 +74,8 @@ golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= diff --git a/configserver/pom.xml b/configserver/pom.xml index fedeec61fb3..c750e4fe0f6 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -95,7 +95,7 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.github.tomakehurst</groupId> + <groupId>org.wiremock</groupId> <artifactId>wiremock-standalone</artifactId> <scope>test</scope> </dependency> diff --git a/container-core/pom.xml b/container-core/pom.xml index 5c6a85a1279..8c30dad7e22 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -373,7 +373,7 @@ <!-- TEST scope --> <dependency> - <groupId>com.github.tomakehurst</groupId> + <groupId>org.wiremock</groupId> <artifactId>wiremock-standalone</artifactId> <scope>test</scope> </dependency> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 0120a6c82fe..96aa120d42f 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -33,6 +33,10 @@ <groupId>org.jvnet.hudson</groupId> <artifactId>annotation-indexer</artifactId> </exclusion> + <exclusion> + <groupId>javax.activation</groupId> + <artifactId>javax.activation-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 6f6b0fc2b79..eca0c8058a1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -121,7 +121,7 @@ public class Dispatcher extends AbstractComponent { DispatchNodesConfig nodesConfig, VipStatus vipStatus, InvokerFactoryFactory invokerFactories) { this(dispatchConfig, rpcConnectionPool, new SearchCluster(clusterId.stringValue(), dispatchConfig.minActivedocsPercentage(), - toNodes(nodesConfig), vipStatus, new RpcPingFactory(rpcConnectionPool)), + toNodes(clusterId.stringValue(), nodesConfig), vipStatus, new RpcPingFactory(rpcConnectionPool)), invokerFactories); } @@ -180,7 +180,7 @@ public class Dispatcher extends AbstractComponent { * under the assumption that this is the common case, i.e., new nodes have no documents yet. */ void updateWithNewConfig(DispatchNodesConfig nodesConfig) { - try (var items = volatileItems()) { // Marking a reference to the old snapshot, which we want to have cleaned up. + try (var items = volatileItems()) { // Mark a reference to the old snapshot, which we want to have cleaned up. items.get().countDown(); // Decrement for its initial creation reference, so it may reach 0. // Let the RPC pool know about the new nodes, and set up the delayed cleanup that we need to do. @@ -192,7 +192,7 @@ public class Dispatcher extends AbstractComponent { }; // Update the nodes the search cluster keeps track of, and what nodes are monitored. - ClusterMonitor<Node> newMonitor = searchCluster.updateNodes(toNodes(nodesConfig), dispatchConfig.minActivedocsPercentage()); + ClusterMonitor<Node> newMonitor = searchCluster.updateNodes(toNodes(searchCluster.name(), nodesConfig), dispatchConfig.minActivedocsPercentage()); // Update the snapshot to use the new nodes set in the search cluster; the RPC pool is ready for this. this.volatileItems = update(newMonitor); @@ -234,9 +234,9 @@ public class Dispatcher extends AbstractComponent { case LATENCY_AMORTIZED_OVER_TIME -> LoadBalancer.Policy.LATENCY_AMORTIZED_OVER_TIME; }; } - private static List<Node> toNodes(DispatchNodesConfig nodesConfig) { + private static List<Node> toNodes(String clusterName, DispatchNodesConfig nodesConfig) { return nodesConfig.node().stream() - .map(n -> new Node(n.key(), n.host(), n.group())) + .map(n -> new Node(clusterName, n.key(), n.host(), n.group())) .toList(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index aeb04bfb141..31e02f910ee 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -12,6 +12,7 @@ import java.util.concurrent.atomic.AtomicLong; */ public class Node { + private final String clusterName; private final int key; private final String hostname; private final int group; @@ -25,7 +26,8 @@ public class Node { private volatile boolean working = true; private volatile boolean isBlockingWrites = false; - public Node(int key, String hostname, int group) { + public Node(String clusterName, int key, String hostname, int group) { + this.clusterName = clusterName; this.key = key; this.hostname = hostname; this.group = group; @@ -33,7 +35,7 @@ public class Node { /** Give a monotonically increasing sequence number.*/ public long createPingSequenceId() { return pingSequence.incrementAndGet(); } - /** Checks if this pong is received in line and accepted, or out of band and should be ignored..*/ + /** Checks if this pong is received in line and accepted, or out of band and should be ignored. */ public boolean isLastReceivedPong(long pingId ) { long last = lastPong.get(); while ((pingId > last) && ! lastPong.compareAndSet(last, pingId)) { @@ -103,8 +105,8 @@ public class Node { @Override public String toString() { - return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + - " statusIsKnown = " + statusIsKnown + " working = " + working + + return "search node in cluster = " + clusterName + " key = " + key + " hostname = "+ hostname + + " path = " + pathIndex + " in group " + group + " statusIsKnown = " + statusIsKnown + " working = " + working + " activeDocs = " + getActiveDocuments() + " targetActiveDocs = " + getTargetActiveDocuments(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 3c8950f1f7f..59b4637a627 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -8,9 +8,9 @@ import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.cluster.NodeManager; import com.yahoo.yolean.UncheckedInterruptedException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -18,6 +18,7 @@ import java.util.concurrent.Executor; import java.util.logging.Logger; import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toMap; /** * A model of a search cluster we might want to dispatch queries to. @@ -42,7 +43,7 @@ public class SearchCluster implements NodeManager<Node> { * if it only queries this cluster when the local node cannot be used, to avoid unnecessary * cross-node network traffic. */ - private final Node localCorpusDispatchTarget; + private volatile Node localCorpusDispatchTarget; public SearchCluster(String clusterId, double minActivedocsPercentage, Collection<Node> nodes, VipStatus vipStatus, PingFactory pingFactory) { @@ -62,12 +63,14 @@ public class SearchCluster implements NodeManager<Node> { /** Sets the new nodes to monitor to be the new nodes, but keep any existing node instances which equal the new ones. */ public ClusterMonitor<Node> updateNodes(Collection<Node> newNodes, double minActivedocsPercentage) { - Collection<Node> retainedNodes = groups.nodes(); - Collection<Node> currentNodes = new HashSet<>(newNodes); - retainedNodes.retainAll(currentNodes); // Throw away all old nodes which are not in the new set. - currentNodes.removeIf(retainedNodes::contains); // Throw away all new nodes for which we have more information in an old object. - Collection<Node> addedNodes = List.copyOf(currentNodes); - currentNodes.addAll(retainedNodes); // Keep the old nodes that were replaced in the new set. + List<Node> currentNodes = new ArrayList<>(newNodes); + List<Node> addedNodes = new ArrayList<>(); + Map<Node, Node> retainedNodes = groups.nodes().stream().collect(toMap(node -> node, node -> node)); + for (int i = 0; i < currentNodes.size(); i++) { + Node retained = retainedNodes.get(currentNodes.get(i)); + if (retained != null) currentNodes.set(i, retained); + else addedNodes.add(currentNodes.get(i)); + } SearchGroupsImpl groups = toGroups(currentNodes, minActivedocsPercentage); ClusterMonitor<Node> monitor = new ClusterMonitor<>(this, false); for (Node node : groups.nodes()) monitor.add(node, true); @@ -75,6 +78,7 @@ public class SearchCluster implements NodeManager<Node> { try { while (addedNodes.stream().anyMatch(node -> node.isWorking() == null)) { Thread.sleep(1); } } catch (InterruptedException e) { throw new UncheckedInterruptedException(e, true); } pingIterationCompleted(groups); + this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), groups); this.groups = groups; return monitor; } @@ -139,6 +143,7 @@ public class SearchCluster implements NodeManager<Node> { } private void updateWorkingState(Node node, boolean isWorking) { + log.fine(() -> "Updating working state of " + node + " to " + isWorking); node.setWorking(isWorking); updateVipStatusOnNodeChange(node, isWorking); } @@ -214,6 +219,7 @@ public class SearchCluster implements NodeManager<Node> { /** Used by the cluster monitor to manage node status */ @Override public void ping(ClusterMonitor<Node> clusterMonitor, Node node, Executor executor) { + log.fine(() -> "Pinging " + node); Pinger pinger = pingFactory.createPinger(node, clusterMonitor, new PongCallback(node, clusterMonitor)); pinger.ping(); } @@ -300,6 +306,7 @@ public class SearchCluster implements NodeManager<Node> { @Override public void handle(Pong pong) { + log.fine(() -> "Got pong from " + node + ": " + pong); if (pong.badResponse()) { clusterMonitor.failed(node, pong.error().get()); } else { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 7a63eb07641..fb483a8eb7b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -30,7 +30,6 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -83,7 +82,7 @@ public class FastSearcherTestCase { @Test void testSinglePassGroupingIsForcedWithSingleNodeGroups() { FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(List.of(new Node(0, "host0", 0))), + MockDispatcher.create(List.of(new Node("test", 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), documentdbInfoConfig("test"), @@ -106,7 +105,7 @@ public class FastSearcherTestCase { @Test void testRankProfileValidation() { FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(List.of(new Node(0, "host0", 0))), + MockDispatcher.create(List.of(new Node("test", 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), documentdbInfoConfig("test"), @@ -125,7 +124,7 @@ public class FastSearcherTestCase { .setHasSummaryFeatures(false) .build()); FastSearcher backend = new FastSearcher("container.0", - MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 0))), + MockDispatcher.create(Collections.singletonList(new Node("test", 0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), documentDb, @@ -142,7 +141,7 @@ public class FastSearcherTestCase { @Test void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { - MockDispatcher dispatcher = MockDispatcher.create(List.of(new Node(0, "host0", 0), new Node(2, "host1", 0))); + MockDispatcher dispatcher = MockDispatcher.create(List.of(new Node("test", 0, "host0", 0), new Node("test", 2, "host1", 0))); FastSearcher fastSearcher = new FastSearcher("container.0", dispatcher, @@ -184,7 +183,7 @@ public class FastSearcherTestCase { searchClusterB.name(clusterName); b.searchcluster(searchClusterB); VipStatus vipStatus = new VipStatus(b.build()); - List<Node> nodes_1 = List.of(new Node(0, "host0", 0)); + List<Node> nodes_1 = List.of(new Node("test", 0, "host0", 0)); RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(), MockDispatcher.toNodesConfig(nodes_1)); MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); dispatch_1.clusterMonitor.shutdown(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 1278afe3759..3397638b950 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -77,7 +77,7 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 0, 0) { @Override public Optional<Node> localCorpusDispatchTarget() { - return Optional.of(new Node(1, "test", 1)); + return Optional.of(new Node("test", 1, "test", 1)); } }; MockInvokerFactory invokerFactory = new MockInvokerFactory(cl.groupList(), dispatchConfig, (n, a) -> true); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 688cdffe22d..500201df26f 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -238,8 +238,8 @@ public class InterleavedSearchInvokerTest { @Test void requireThatTopKProbabilityOverrideIsDisabledOnContentSkew() throws IOException { - Node node0 = new Node(0, "host0", 0); - Node node1 = new Node(1, "host1", 0); + Node node0 = new Node("test", 0, "host0", 0); + Node node1 = new Node("test", 1, "host1", 0); Group group = new Group(0, List.of(node0, node1)); node0.setActiveDocuments(1000000); @@ -250,8 +250,8 @@ public class InterleavedSearchInvokerTest { @Test void requireThatTopKProbabilityOverrideIsDisabledOnLittleContent() throws IOException { - Node node0 = new Node(0, "host0", 0); - Node node1 = new Node(1, "host1", 0); + Node node0 = new Node("test", 0, "host0", 0); + Node node1 = new Node("test", 1, "host1", 0); Group group = new Group(0, List.of(node0, node1)); node0.setActiveDocuments(10); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 4956698cc2f..b57d97ebb84 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -28,7 +28,7 @@ public class LoadBalancerTest { private static final double delta = 0.0000001; @Test void requireThatLoadBalancerServesSingleNodeSetups() { - Node n1 = new Node(0, "test-node1", 0); + Node n1 = new Node("test", 0, "test-node1", 0); LoadBalancer lb = new LoadBalancer(List.of(new Group(0, List.of(n1))), LoadBalancer.Policy.ROUNDROBIN); Optional<Group> grp = lb.takeGroup(null); @@ -40,8 +40,8 @@ public class LoadBalancerTest { @Test void requireThatLoadBalancerServesMultiGroupSetups() { - Node n1 = new Node(0, "test-node1", 0); - Node n2 = new Node(1, "test-node2", 1); + Node n1 = new Node("test", 0, "test-node1", 0); + Node n2 = new Node("test", 1, "test-node2", 1); LoadBalancer lb = new LoadBalancer(List.of(new Group(0, List.of(n1)), new Group(1,List.of(n2))), LoadBalancer.Policy.ROUNDROBIN); Optional<Group> grp = lb.takeGroup(null); @@ -53,10 +53,10 @@ public class LoadBalancerTest { @Test void requireThatLoadBalancerServesClusteredGroups() { - Node n1 = new Node(0, "test-node1", 0); - Node n2 = new Node(1, "test-node2", 0); - Node n3 = new Node(0, "test-node3", 1); - Node n4 = new Node(1, "test-node4", 1); + Node n1 = new Node("test", 0, "test-node1", 0); + Node n2 = new Node("test", 1, "test-node2", 0); + Node n3 = new Node("test", 0, "test-node3", 1); + Node n4 = new Node("test", 1, "test-node4", 1); LoadBalancer lb = new LoadBalancer(List.of(new Group(0, List.of(n1,n2)), new Group(1,List.of(n3,n4))), LoadBalancer.Policy.ROUNDROBIN); Optional<Group> grp = lb.takeGroup(null); @@ -65,8 +65,8 @@ public class LoadBalancerTest { @Test void requireThatLoadBalancerReturnsDifferentGroups() { - Node n1 = new Node(0, "test-node1", 0); - Node n2 = new Node(1, "test-node2", 1); + Node n1 = new Node("test", 0, "test-node1", 0); + Node n2 = new Node("test", 1, "test-node2", 1); LoadBalancer lb = new LoadBalancer(List.of(new Group(0, List.of(n1)), new Group(1,List.of(n2))), LoadBalancer.Policy.ROUNDROBIN); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index aca84386af7..b47015c08c6 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -3,17 +3,13 @@ package com.yahoo.search.dispatch; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; -import java.io.IOException; import java.util.List; import java.util.Optional; -import java.util.OptionalInt; class MockInvoker extends SearchInvoker { @@ -23,7 +19,7 @@ class MockInvoker extends SearchInvoker { int hitsRequested; protected MockInvoker(int key, Coverage coverage) { - super(Optional.of(new Node(key, "?", 0))); + super(Optional.of(new Node("test", key, "?", 0))); this.coverage = coverage; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index 7c1e7372507..b877bac5d74 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -33,7 +33,7 @@ public class RpcSearchInvokerTest { var lengthHolder = new AtomicInteger(); var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); - var invoker = new RpcSearchInvoker(mockSearcher(), compressor, new Node(7, "seven", 1), mockPool, 1000); + var invoker = new RpcSearchInvoker(mockSearcher(), compressor, new Node("test", 7, "seven", 1), mockPool, 1000); Query q = new Query("search/?query=test&hits=10&offset=3"); RpcSearchInvoker.RpcContext context = (RpcSearchInvoker.RpcContext) invoker.sendSearchRequest(q, null); @@ -47,7 +47,7 @@ public class RpcSearchInvokerTest { assertEquals(3, request.getOffset()); assertTrue(request.getQueryTreeBlob().size() > 0); - var invoker2 = new RpcSearchInvoker(mockSearcher(), compressor, new Node(8, "eight", 1), mockPool, 1000); + var invoker2 = new RpcSearchInvoker(mockSearcher(), compressor, new Node("test", 8, "eight", 1), mockPool, 1000); RpcSearchInvoker.RpcContext context2 = (RpcSearchInvoker.RpcContext) invoker2.sendSearchRequest(q, context); assertSame(context, context2); assertEquals(lengthHolder.get(), context.compressedPayload.uncompressedSize()); @@ -62,7 +62,7 @@ public class RpcSearchInvokerTest { var lengthHolder = new AtomicInteger(); var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); - var invoker = new RpcSearchInvoker(mockSearcher(), compressor, new Node(7, "seven", 1), mockPool, maxHits); + var invoker = new RpcSearchInvoker(mockSearcher(), compressor, new Node("test", 7, "seven", 1), mockPool, maxHits); Query q = new Query("search/?query=test&hits=10&offset=3"); invoker.sendSearchRequest(q, null); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/MockSearchCluster.java index cd0791a3881..6900cc5dd52 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/MockSearchCluster.java @@ -79,7 +79,7 @@ public class MockSearchCluster extends SearchCluster { for (int group = 0; group < numGroups; group++) { List<Node> groupNodes = new ArrayList<>(); for (int i = 0; i < nodesPerGroup; i++) { - Node node = new Node(distributionKey, "host" + distributionKey, group); + Node node = new Node("test", distributionKey, "host" + distributionKey, group); node.setWorking(true); groupNodes.add(node); distributionKey++; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index bfe1aed1084..f6a1ca5cae3 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -19,8 +19,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static java.util.function.Function.identity; @@ -58,7 +56,7 @@ public class SearchClusterTest { for (String name : nodeNames) { int key = nodes.size() % nodesPerGroup; int group = nodes.size() / nodesPerGroup; - nodes.add(new Node(key, name, group)); + nodes.add(new Node("test", key, name, group)); numDocsPerNode.add(new AtomicInteger(1)); pingCounts.add(new AtomicInteger(0)); } @@ -326,7 +324,7 @@ public class SearchClusterTest { @Test void requireThatPingSequenceIsUpHeld() { - Node node = new Node(1, "n", 1); + Node node = new Node("test", 1, "n", 1); assertEquals(1, node.createPingSequenceId()); assertEquals(2, node.createPingSequenceId()); assertEquals(0, node.getLastReceivedPongId()); @@ -348,7 +346,7 @@ public class SearchClusterTest { @Test void requireThatSingleNodeGroupIsInBalance() { - Group group = new Group(0, List.of(new Node(1, "n", 1))); + Group group = new Group(0, List.of(new Node("test", 1, "n", 1))); group.nodes().forEach(node -> node.setWorking(true)); assertTrue(group.isBalanced()); group.aggregateNodeValues(); @@ -360,7 +358,7 @@ public class SearchClusterTest { @Test void requireThatMultiNodeGroupDetectsBalance() { - Group group = new Group(0, List.of(new Node(1, "n1", 1), new Node(2, "n2", 1))); + Group group = new Group(0, List.of(new Node("test", 1, "n1", 1), new Node("test", 2, "n2", 1))); assertTrue(group.isBalanced()); group.nodes().forEach(node -> node.setWorking(true)); assertTrue(group.isBalanced()); @@ -386,33 +384,39 @@ public class SearchClusterTest { @Test void requireThatPreciselyTheRetainedNodesAreKeptWhenNodesAreUpdated() { try (State state = new State("query", 2, IntStream.range(0, 6).mapToObj(i -> "node-" + i).toList())) { - List<Node> referenceNodes = List.of(new Node(0, "node-0", 0), - new Node(1, "node-1", 0), - new Node(0, "node-2", 1), - new Node(1, "node-3", 1), - new Node(0, "node-4", 2), - new Node(1, "node-5", 2)); + List<Node> referenceNodes = List.of(new Node("test", 0, "node-0", 0), + new Node("test", 1, "node-1", 0), + new Node("test", 0, "node-2", 1), + new Node("test", 1, "node-3", 1), + new Node("test", 0, "node-4", 2), + new Node("test", 1, "node-5", 2)); SearchGroups oldGroups = state.searchCluster.groupList(); assertEquals(Set.copyOf(referenceNodes), oldGroups.nodes()); - List<Node> updatedNodes = List.of(new Node(0, "node-1", 0), // Swap node-0 and node-1 - new Node(1, "node-0", 0), // Swap node-1 and node-0 - new Node(0, "node-4", 1), // Swap node-2 and node-4 - new Node(1, "node-3", 1), - new Node(0, "node-2", 2), // Swap node-4 and node-2 - new Node(1, "node-6", 2)); // Replace node-6 + List<Node> updatedNodes = List.of(new Node("test", 0, "node-1", 0), // Swap node-0 and node-1 + new Node("test", 1, "node-0", 0), // Swap node-1 and node-0 + new Node("test", 0, "node-4", 1), // Swap node-2 and node-4 + new Node("test", 1, "node-3", 1), + new Node("test", 0, "node-2", 2), // Swap node-4 and node-2 + new Node("test", 1, "node-6", 2)); // Replace node-6 state.searchCluster.updateNodes(updatedNodes, 100.0); SearchGroups newGroups = state.searchCluster.groupList(); assertEquals(Set.copyOf(updatedNodes), newGroups.nodes()); - Map<Node, Node> oldNodesByIdentity = newGroups.nodes().stream().collect(toMap(identity(), identity())); + Map<Node, Node> oldNodesByIdentity = oldGroups.nodes().stream().collect(toMap(identity(), identity())); Map<Node, Node> newNodesByIdentity = newGroups.nodes().stream().collect(toMap(identity(), identity())); assertSame(updatedNodes.get(0), newNodesByIdentity.get(updatedNodes.get(0))); assertSame(updatedNodes.get(1), newNodesByIdentity.get(updatedNodes.get(1))); assertSame(updatedNodes.get(2), newNodesByIdentity.get(updatedNodes.get(2))); - assertSame(oldNodesByIdentity.get(referenceNodes.get(3)), newNodesByIdentity.get(updatedNodes.get(3))); + assertSame(oldNodesByIdentity.get(updatedNodes.get(3)), newNodesByIdentity.get(updatedNodes.get(3))); assertSame(updatedNodes.get(4), newNodesByIdentity.get(updatedNodes.get(4))); assertSame(updatedNodes.get(5), newNodesByIdentity.get(updatedNodes.get(5))); + + // Also verify search-path index within group follows node order, as given by config. + int[] pathIndexWithinGroup = new int[3]; + for (Node node : updatedNodes) + assertEquals(pathIndexWithinGroup[node.group()]++, newNodesByIdentity.get(node).pathIndex(), + "search path index within group should follow updated node order for: " + node); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java index 9c29f5c30f4..664669d8e55 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java @@ -122,7 +122,7 @@ public class Bill { public BigDecimal sumAdditionalCost() { // anything that is not covered by the cost for resources is "additional" costs - var resourceCosts = sumCpuCost().add(sumMemoryCost()).add(sumDiskCost()); + var resourceCosts = sumCpuCost().add(sumMemoryCost()).add(sumDiskCost()).add(sumGpuCost()); return sum().subtract(resourceCosts); } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 8104647c2b8..3c3c93735e7 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -190,7 +190,7 @@ </dependency> <dependency> - <groupId>com.github.tomakehurst</groupId> + <groupId>org.wiremock</groupId> <artifactId>wiremock-standalone</artifactId> <scope>test</scope> </dependency> diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 4d63810cd4b..21d132bb574 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -42,7 +42,7 @@ <javax.inject.vespa.version>1</javax.inject.vespa.version> <javax.servlet-api.vespa.version>3.1.0</javax.servlet-api.vespa.version> <javax.ws.rs-api.vespa.version>2.1.1</javax.ws.rs-api.vespa.version> - <jaxb-api.vespa.version>2.3.0</jaxb-api.vespa.version> + <jaxb-api.vespa.version>2.3.1</jaxb-api.vespa.version> <jaxb-core.vespa.version>2.3.0.1</jaxb-core.vespa.version> <jaxb-impl.vespa.version>2.3.0</jaxb-impl.vespa.version> <slf4j.vespa.version>1.7.36</slf4j.vespa.version> @@ -76,6 +76,7 @@ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version> + <byte-buddy.vespa.version>1.14.7</byte-buddy.vespa.version> <checker-qual.vespa.version>3.37.0</checker-qual.vespa.version> <commons-codec.vespa.version>1.16.0</commons-codec.vespa.version> <commons-csv.vespa.version>1.10.0</commons-csv.vespa.version> @@ -121,7 +122,7 @@ <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.16.0</prometheus.client.vespa.version> <protobuf.vespa.version>3.24.2</protobuf.vespa.version> - <questdb.vespa.version>6.7</questdb.vespa.version> + <questdb.vespa.version>7.3.1</questdb.vespa.version> <spifly.vespa.version>1.3.6</spifly.vespa.version> <snappy.vespa.version>1.1.10.3</snappy.vespa.version> <surefire.vespa.version>3.1.2</surefire.vespa.version> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index ea716556210..528223bffa7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -191,13 +191,13 @@ public class Flags { // TODO: Move to a permanent flag public static final UnboundListFlag<String> ALLOWED_ATHENZ_PROXY_IDENTITIES = defineListFlag( "allowed-athenz-proxy-identities", List.of(), String.class, - List.of("bjorncs", "tokle"), "2021-02-10", "2023-09-01", + List.of("bjorncs", "tokle"), "2021-02-10", "2023-10-01", "Allowed Athenz proxy identities", "takes effect at redeployment"); public static final UnboundIntFlag MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS = defineIntFlag( "max-activation-inhibited-out-of-sync-groups", 0, - List.of("vekterli"), "2021-02-19", "2023-09-01", + List.of("vekterli"), "2021-02-19", "2023-12-01", "Allows replicas in up to N content groups to not be activated " + "for query visibility if they are out of sync with a majority of other replicas", "Takes effect at redeployment", @@ -205,7 +205,7 @@ public class Flags { public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, - List.of("geirst", "vekterli"), "2021-07-16", "2023-09-01", + List.of("geirst", "vekterli"), "2021-07-16", "2023-12-01", "Minimum ratio of nodes that have to be available (i.e. not Down) in any hierarchic content cluster group for the group to be Up", "Takes effect at redeployment", APPLICATION_ID); @@ -230,7 +230,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLED_HORIZON_DASHBOARD = defineFeatureFlag( "enabled-horizon-dashboard", false, - List.of("olaa"), "2021-09-13", "2023-09-01", + List.of("olaa"), "2021-09-13", "2023-12-01", "Enable Horizon dashboard", "Takes effect immediately", TENANT_ID, CONSOLE_USER_EMAIL @@ -252,7 +252,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, - List.of("tokle"), "2022-05-09", "2023-09-01", + List.of("tokle"), "2022-05-09", "2023-10-01", "Enable or disable proxy protocol mixed mode", "Takes effect on redeployment", APPLICATION_ID); @@ -266,7 +266,7 @@ public class Flags { public static final UnboundBooleanFlag SEPARATE_METRIC_CHECK_CONFIG = defineFeatureFlag( "separate-metric-check-config", false, - List.of("olaa"), "2022-07-04", "2023-09-01", + List.of("olaa"), "2022-07-04", "2023-12-01", "Determines whether one metrics config check should be written per Vespa node", "Takes effect on next tick", HOSTNAME); @@ -281,7 +281,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_OTELCOL = defineFeatureFlag( "enable-otel-collector", false, - List.of("olaa"), "2022-09-23", "2023-09-01", + List.of("olaa"), "2022-09-23", "2023-12-01", "Whether an OpenTelemetry collector should be enabled", "Takes effect at next tick", APPLICATION_ID); @@ -317,14 +317,14 @@ public class Flags { "Takes effect at next host-admin tick"); public static final UnboundListFlag<String> WEIGHTED_ENDPOINT_RECORD_TTL = defineListFlag( - "weighted-endpoint-record-ttl", List.of(), String.class, List.of("jonmv"), "2023-05-16", "2023-09-01", + "weighted-endpoint-record-ttl", List.of(), String.class, List.of("jonmv"), "2023-05-16", "2024-01-01", "A list of endpoints and custom TTLs, on the form \"endpoint-fqdn:TTL-seconds\". " + "Where specified, CNAME records are used instead of the default ALIAS records, which have a default 60s TTL.", "Takes effect at redeployment from controller"); public static final UnboundBooleanFlag ENABLE_DATAPLANE_PROXY = defineFeatureFlag( "enable-dataplane-proxy", false, - List.of("mortent", "olaa"), "2023-05-15", "2023-09-01", + List.of("mortent", "olaa"), "2023-05-15", "2023-10-01", "Whether to enable dataplane proxy", "Takes effect at redeployment", APPLICATION_ID diff --git a/http-client/pom.xml b/http-client/pom.xml index 661cc68f2b7..e5900d28009 100644 --- a/http-client/pom.xml +++ b/http-client/pom.xml @@ -48,7 +48,7 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.github.tomakehurst</groupId> + <groupId>org.wiremock</groupId> <artifactId>wiremock-standalone</artifactId> <scope>test</scope> </dependency> diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 7ec1f645ad9..b757c01f57b 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -67,7 +67,7 @@ org.vafer:jdependency:2.8.0 #[test-only] # Contains dependencies that are used exclusively in 'test' scope junit:junit:4.13.2 -net.bytebuddy:byte-buddy:1.14.6 +net.bytebuddy:byte-buddy:1.14.7 net.bytebuddy:byte-buddy-agent:1.14.6 org.apiguardian:apiguardian-api:1.1.2 org.hamcrest:hamcrest:2.2 diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index f67989afd76..af0dbaa83fb 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -111,7 +111,7 @@ <dependency> - <groupId>com.github.tomakehurst</groupId> + <groupId>org.wiremock</groupId> <artifactId>wiremock-standalone</artifactId> <scope>test</scope> </dependency> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index ac915e3fcf6..c85af8e4965 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -14,11 +14,13 @@ import io.questdb.cairo.CairoException; import io.questdb.cairo.DefaultCairoConfiguration; import io.questdb.cairo.TableToken; import io.questdb.cairo.TableWriter; +import io.questdb.cairo.security.AllowAllSecurityContext; import io.questdb.cairo.sql.Record; import io.questdb.cairo.sql.RecordCursor; import io.questdb.cairo.sql.RecordCursorFactory; import io.questdb.griffin.CompiledQuery; import io.questdb.griffin.SqlCompiler; +import io.questdb.griffin.SqlCompilerFactoryImpl; import io.questdb.griffin.SqlException; import io.questdb.griffin.SqlExecutionContext; import io.questdb.griffin.SqlExecutionContextImpl; @@ -83,7 +85,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { this.dataDir = dataDir; engine = new CairoEngine(new DefaultCairoConfiguration(dataDir)); - sqlCompilerPool = new ConcurrentResourcePool<>(() -> new SqlCompiler(engine())); + sqlCompilerPool = new ConcurrentResourcePool<>(() -> SqlCompilerFactoryImpl.INSTANCE.getInstance(engine())); nodeTable = new Table(dataDir, "metrics"); clusterTable = new Table(dataDir, "clusterMetrics"); ensureTablesExist(); @@ -234,7 +236,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { private void ensureClusterTableIsUpdated() { try { - if (0 == engine().getStatus(newContext().getCairoSecurityContext(), new Path(), clusterTable.token())) { + if (0 == engine().getTableStatus(new Path(), clusterTable.token())) { // Example: clusterTable.ensureColumnExists("write_rate", "float"); } } catch (Exception e) { @@ -353,7 +355,9 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } private SqlExecutionContext newContext() { - return new SqlExecutionContextImpl(engine(), 1); + CairoEngine engine = engine(); + return new SqlExecutionContextImpl(engine, 1) + .with(AllowAllSecurityContext.INSTANCE, null); } /** A questDb table */ @@ -371,16 +375,16 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { // https://stackoverflow.com/questions/67785629/what-does-max-txn-txn-inflight-limit-reached-in-questdb-and-how-to-i-avoid-it new File(dir + "/_txn_scoreboard").delete(); } - private TableToken token() { return engine().getTableToken(name); } + private TableToken token() { return engine().getTableTokenIfExists(name); } boolean exists() { TableToken token = engine().getTableTokenIfExists(name); if (token == null) return false; - return 0 == engine().getStatus(newContext().getCairoSecurityContext(), new Path(), token); + return 0 == engine().getTableStatus(new Path(), token); } TableWriter getWriter() { - return engine().getWriter(newContext().getCairoSecurityContext(), token(), "getWriter"); + return engine().getWriter(token(), "getWriter"); } void gc() { diff --git a/parent/pom.xml b/parent/pom.xml index c92927437d0..31b7eaa8025 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -307,7 +307,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.4.2</version> + <version>5.5.0</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> @@ -504,7 +504,7 @@ <version>3.1.9</version> </dependency> <dependency> - <groupId>com.github.tomakehurst</groupId> + <groupId>org.wiremock</groupId> <artifactId>wiremock-standalone</artifactId> <version>${wiremock.vespa.version}</version> </dependency> @@ -635,6 +635,11 @@ <version>${joda-time.vespa.version}</version> </dependency> <dependency> + <groupId>net.bytebuddy</groupId> + <artifactId>byte-buddy</artifactId> + <version>${byte-buddy.vespa.version}</version> + </dependency> + <dependency> <groupId>net.openhft</groupId> <artifactId>zero-allocation-hashing</artifactId> <version>${zero-allocation-hashing.vespa.version}</version> diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 0b498a791b3..e558074f724 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -1033,20 +1033,23 @@ TEST_F("require that underlying components are explorable", StoreOnlyExplorerFix { assertExplorer({}, f._explorer); EXPECT_TRUE(f._explorer.get_child("attribute").get() == nullptr); + EXPECT_TRUE(f._explorer.get_child("attributewriter").get() == nullptr); EXPECT_TRUE(f._explorer.get_child("index").get() == nullptr); } TEST_F("require that underlying components are explorable", FastAccessExplorerFixture) { - assertExplorer({"attribute"}, f._explorer); + assertExplorer({"attribute", "attributewriter"}, f._explorer); EXPECT_TRUE(f._explorer.get_child("attribute").get() != nullptr); + EXPECT_TRUE(f._explorer.get_child("attributewriter").get() != nullptr); EXPECT_TRUE(f._explorer.get_child("index").get() == nullptr); } TEST_F("require that underlying components are explorable", SearchableExplorerFixture) { - assertExplorer({"attribute", "index"}, f._explorer); + assertExplorer({"attribute", "attributewriter", "index"}, f._explorer); EXPECT_TRUE(f._explorer.get_child("attribute").get() != nullptr); + EXPECT_TRUE(f._explorer.get_child("attributewriter").get() != nullptr); EXPECT_TRUE(f._explorer.get_child("index").get() != nullptr); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index 70a91b418a9..7e5d3accbc3 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_library(searchcore_attribute STATIC SOURCES address_space_usage_stats.cpp attribute_aspect_delayer.cpp - attribute_collection_spec_factory.cpp attribute_collection_spec.cpp + attribute_collection_spec_factory.cpp attribute_config_inspector.cpp attribute_directory.cpp attribute_executor.cpp @@ -24,9 +24,10 @@ vespa_add_library(searchcore_attribute STATIC attribute_usage_stats.cpp attribute_vector_explorer.cpp attribute_writer.cpp - attributes_initializer_base.cpp + attribute_writer_explorer.cpp attributedisklayout.cpp attributemanager.cpp + attributes_initializer_base.cpp attributesconfigscout.cpp document_field_extractor.cpp document_field_populator.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp new file mode 100644 index 00000000000..0e26dcdfa21 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp @@ -0,0 +1,68 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_writer.h" +#include "attribute_writer_explorer.h" +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/vespalib/data/slime/cursor.h> + +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::attribute::Config; +using vespalib::slime::Cursor; +using vespalib::slime::Inserter; + +namespace proton { + +AttributeWriterExplorer::AttributeWriterExplorer(std::shared_ptr<IAttributeWriter> writer) + : _writer(std::move(writer)) +{ +} + +AttributeWriterExplorer::~AttributeWriterExplorer() = default; + +namespace { + +vespalib::string +type_to_string(const Config& cfg) +{ + if (cfg.basicType().type() == BasicType::TENSOR) { + return cfg.tensorType().to_spec(); + } + if (cfg.collectionType().type() == CollectionType::SINGLE) { + return cfg.basicType().asString(); + } + return vespalib::string(cfg.collectionType().asString()) + + "<" + vespalib::string(cfg.basicType().asString()) + ">"; +} + +void +convert_to_slime(const AttributeWriter::WriteContext& context, Cursor& object) +{ + object.setLong("executor_id", context.getExecutorId().getId()); + Cursor& fields = object.setArray("attribute_fields"); + for (const auto& field : context.getFields()) { + Cursor& f = fields.addObject(); + f.setString("name", field.getAttribute().getName()); + f.setString("type", type_to_string(field.getAttribute().getConfig())); + } +} + +} + +void +AttributeWriterExplorer::get_state(const Inserter& inserter, bool full) const +{ + Cursor& object = inserter.insertObject(); + if (full) { + auto* writer = dynamic_cast<AttributeWriter*>(_writer.get()); + if (writer) { + Cursor& contexts = object.setArray("write_contexts"); + for (const auto& context : writer->get_write_contexts()) { + convert_to_slime(context, contexts.addObject()); + } + } + } +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.h new file mode 100644 index 00000000000..5569c2ae132 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.h @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_attribute_writer.h" +#include <vespa/vespalib/net/http/state_explorer.h> + +namespace proton { + +/** + * Class used to explore the state of an attribute writer and its write contexts. + */ +class AttributeWriterExplorer : public vespalib::StateExplorer { +private: + std::shared_ptr<IAttributeWriter> _writer; + +public: + AttributeWriterExplorer(std::shared_ptr<IAttributeWriter> writer); + ~AttributeWriterExplorer(); + + void get_state(const vespalib::slime::Inserter& inserter, bool full) const override; +}; + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp index 8049e76d4b6..cc657b04459 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_subdb_explorer.h" - #include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h> -#include <vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h> +#include <vespa/searchcore/proton/attribute/attribute_writer_explorer.h> #include <vespa/searchcore/proton/docsummary/document_store_explorer.h> +#include <vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h> #include <vespa/searchcorespi/index/index_manager_explorer.h> using searchcorespi::IndexManagerExplorer; @@ -18,6 +18,7 @@ namespace { const vespalib::string DOCUMENT_META_STORE = "documentmetastore"; const vespalib::string DOCUMENT_STORE = "documentstore"; const vespalib::string ATTRIBUTE = "attribute"; +const vespalib::string ATTRIBUTE_WRITER = "attributewriter"; const vespalib::string INDEX = "index"; } @@ -38,10 +39,13 @@ std::vector<vespalib::string> DocumentSubDBExplorer::get_children_names() const { std::vector<vespalib::string> children = {DOCUMENT_META_STORE, DOCUMENT_STORE}; - if (_subDb.getAttributeManager().get() != nullptr) { + if (_subDb.getAttributeManager()) { children.push_back(ATTRIBUTE); } - if (_subDb.getIndexManager().get() != nullptr) { + if (_subDb.get_attribute_writer()) { + children.push_back(ATTRIBUTE_WRITER); + } + if (_subDb.getIndexManager()) { children.push_back(INDEX); } return children; @@ -53,22 +57,27 @@ DocumentSubDBExplorer::get_child(vespalib::stringref name) const if (name == DOCUMENT_META_STORE) { // TODO(geirst): Avoid const cast by adding const interface to // IDocumentMetaStoreContext as seen from IDocumentSubDB. - return std::unique_ptr<StateExplorer>(new DocumentMetaStoreExplorer( - (const_cast<IDocumentSubDB &>(_subDb)).getDocumentMetaStoreContext().getReadGuard())); + return std::make_unique<DocumentMetaStoreExplorer>( + (const_cast<IDocumentSubDB &>(_subDb)).getDocumentMetaStoreContext().getReadGuard()); } else if (name == DOCUMENT_STORE) { - return std::unique_ptr<StateExplorer>(new DocumentStoreExplorer(_subDb.getSummaryManager())); + return std::make_unique<DocumentStoreExplorer>(_subDb.getSummaryManager()); } else if (name == ATTRIBUTE) { - proton::IAttributeManager::SP attrMgr = _subDb.getAttributeManager(); - if (attrMgr.get() != nullptr) { - return std::unique_ptr<StateExplorer>(new AttributeManagerExplorer(attrMgr)); + auto attrMgr = _subDb.getAttributeManager(); + if (attrMgr) { + return std::make_unique<AttributeManagerExplorer>(attrMgr); + } + } else if (name == ATTRIBUTE_WRITER) { + auto writer = _subDb.get_attribute_writer(); + if (writer) { + return std::make_unique<AttributeWriterExplorer>(std::move(writer)); } } else if (name == INDEX) { - searchcorespi::IIndexManager::SP idxMgr = _subDb.getIndexManager(); - if (idxMgr.get() != nullptr) { - return std::unique_ptr<StateExplorer>(new IndexManagerExplorer(std::move(idxMgr))); + auto idxMgr = _subDb.getIndexManager(); + if (idxMgr) { + return std::make_unique<IndexManagerExplorer>(std::move(idxMgr)); } } - return std::unique_ptr<StateExplorer>(); + return {}; } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 47dd8c03706..140bc4d170c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -282,6 +282,12 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const return tasks; } +std::shared_ptr<IAttributeWriter> +FastAccessDocSubDB::get_attribute_writer() const +{ + return _fastAccessFeedView.get()->getAttributeWriter(); +} + proton::IAttributeManager::SP FastAccessDocSubDB::getAttributeManager() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index 942d42b5ad3..3a6eeef7dac 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -119,6 +119,7 @@ public: applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver, const DocumentSubDBReconfig& prepared_reconfig) override; + std::shared_ptr<IAttributeWriter> get_attribute_writer() const override; std::shared_ptr<IAttributeManager> getAttributeManager() const override; IDocumentRetriever::UP getDocumentRetriever() override; void onReplayDone() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 506af3f6355..92fa0dcad43 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -30,6 +30,7 @@ class DocumentSubDBReconfig; class DocumentSubDbInitializer; class DocumentSubDbInitializerResult; class FeedHandler; +class IAttributeWriter; class IDocumentDBReference; class IDocumentRetriever; class IFeedView; @@ -92,6 +93,7 @@ public: virtual std::shared_ptr<IFeedView> getFeedView() const = 0; virtual void clearViews() = 0; virtual const std::shared_ptr<ISummaryManager> &getSummaryManager() const = 0; + virtual std::shared_ptr<IAttributeWriter> get_attribute_writer() const = 0; virtual std::shared_ptr<IAttributeManager> getAttributeManager() const = 0; virtual const std::shared_ptr<searchcorespi::IIndexManager> &getIndexManager() const = 0; virtual const std::shared_ptr<ISummaryAdapter> &getSummaryAdapter() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 4f694bcfd38..b846a3eceff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -348,6 +348,12 @@ SearchableDocSubDB::clearViews() { Parent::clearViews(); } +std::shared_ptr<IAttributeWriter> +SearchableDocSubDB::get_attribute_writer() const +{ + return _rFeedView.get()->getAttributeWriter(); +} + TransientResourceUsage SearchableDocSubDB::get_transient_resource_usage() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 97172f23bcb..7cfe2b5f444 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -109,6 +109,8 @@ public: void clearViews() override; + std::shared_ptr<IAttributeWriter> get_attribute_writer() const override; + std::shared_ptr<IAttributeManager> getAttributeManager() const override { return _rSearchView.get()->getAttributeManager(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 1c956a4c45c..7a6bca2cd1f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -214,6 +214,7 @@ public: void clearViews() override; const ISummaryManager::SP &getSummaryManager() const override { return _iSummaryMgr; } + std::shared_ptr<IAttributeWriter> get_attribute_writer() const override { return {}; } IAttributeManager::SP getAttributeManager() const override; const std::shared_ptr<searchcorespi::IIndexManager> & getIndexManager() const override; const ISummaryAdapter::SP & getSummaryAdapter() const override { return _summaryAdapter; } diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index d53016d7d8d..12603f1195a 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -66,14 +66,9 @@ struct DummyDocumentSubDb : public IDocumentSubDB IFeedView::SP getFeedView() const override { return IFeedView::SP(); } void clearViews() override {} const ISummaryManager::SP &getSummaryManager() const override { return _summaryManager; } - proton::IAttributeManager::SP getAttributeManager() const override { - return proton::IAttributeManager::SP(); - } - - void validateDocStore(FeedHandler &, SerialNum ) const override { - - } - + std::shared_ptr<IAttributeWriter> get_attribute_writer() const override { return {}; } + proton::IAttributeManager::SP getAttributeManager() const override { return {}; } + void validateDocStore(FeedHandler &, SerialNum ) const override {} const IIndexManager::SP &getIndexManager() const override { return _indexManager; } const ISummaryAdapter::SP &getSummaryAdapter() const override { return _summaryAdapter; } const IIndexWriter::SP &getIndexWriter() const override { return _indexWriter; } diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 64e61410915..ba6985ad690 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -84,11 +84,12 @@ io.prometheus:simpleclient_tracer_otel_agent:0.16.0 jakarta.annotation:jakarta.annotation-api:1.3.5 jakarta.validation:jakarta.validation-api:2.0.2 jakarta.ws.rs:jakarta.ws.rs-api:2.1.6 +javax.activation:javax.activation-api:1.2.0 javax.annotation:javax.annotation-api:1.2 javax.inject:javax.inject:1 javax.servlet:javax.servlet-api:3.1.0 javax.ws.rs:javax.ws.rs-api:2.1.1 -javax.xml.bind:jaxb-api:2.3.0 +javax.xml.bind:jaxb-api:2.3.1 joda-time:joda-time:2.12.5 junit:junit:4.13.2 net.java.dev.jna:jna:5.13.0 @@ -203,7 +204,7 @@ org.ow2.asm:asm-analysis:9.5 org.ow2.asm:asm-commons:9.5 org.ow2.asm:asm-tree:9.5 org.ow2.asm:asm-util:9.5 -org.questdb:questdb:6.7 +org.questdb:questdb:7.3.1 org.slf4j:jcl-over-slf4j:1.7.36 org.slf4j:log4j-over-slf4j:1.7.36 org.slf4j:slf4j-api:1.7.36 @@ -220,8 +221,7 @@ xml-apis:xml-apis:1.4.01 com.google.guava:guava-testlib:32.1.2-jre com.google.inject:guice:4.2.3 com.google.jimfs:jimfs:1.3.0 -net.bytebuddy:byte-buddy:1.12.21 -net.bytebuddy:byte-buddy:1.14.6 +net.bytebuddy:byte-buddy:1.14.7 net.bytebuddy:byte-buddy-agent:1.14.6 org.apache.curator:curator-test:5.5.0 org.assertj:assertj-core:3.24.2 diff --git a/vespalib/src/tests/fastos/file_test.cpp b/vespalib/src/tests/fastos/file_test.cpp index 99c7b858723..a9cec81d084 100644 --- a/vespalib/src/tests/fastos/file_test.cpp +++ b/vespalib/src/tests/fastos/file_test.cpp @@ -181,16 +181,6 @@ TEST(FileTest, ReadWriteTest) { EXPECT_TRUE(std::filesystem::remove(std::filesystem::path(rwFilename))); } -TEST(FileTest, ScanDirectoryTest) { - auto scanDir = std::make_unique<FastOS_DirectoryScan>("."); - while (scanDir->ReadNext()) { - const char *name = scanDir->GetName(); - bool isDirectory = scanDir->IsDirectory(); - bool isRegular = scanDir->IsRegular(); - fprintf(stderr, "%-30s %s\n", name, isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); - } -} - TEST(FileTest, ReadBufTest) { FastOS_File file(roFilename.c_str()); char buffer[20]; diff --git a/vespalib/src/vespa/fastos/file.cpp b/vespalib/src/vespa/fastos/file.cpp index 19a000296de..0c05c1ad894 100644 --- a/vespalib/src/vespa/fastos/file.cpp +++ b/vespalib/src/vespa/fastos/file.cpp @@ -306,10 +306,3 @@ FastOS_FileInterface::getLastErrorString() void FastOS_FileInterface::dropFromCache() const { } - -FastOS_DirectoryScanInterface::FastOS_DirectoryScanInterface(const char *path) - : _searchPath(path) -{ -} - -FastOS_DirectoryScanInterface::~FastOS_DirectoryScanInterface() = default; diff --git a/vespalib/src/vespa/fastos/file.h b/vespalib/src/vespa/fastos/file.h index 079a2b610f1..cd7a22a02b2 100644 --- a/vespalib/src/vespa/fastos/file.h +++ b/vespalib/src/vespa/fastos/file.h @@ -2,8 +2,7 @@ //************************************************************************ /** * @file - * Class definitions for FastOS_File, FastOS_DirectoryScan and - * FastOS_StatInfo. + * Class definitions for FastOS_File and FastOS_StatInfo. * * @author Div, Oivind H. Danielsen */ @@ -544,104 +543,6 @@ public: vespalib::system_time _modifiedTime; }; - -/** - * This class enumerates the contents of a given directory. - * - * Example: - * @code - * void Foo::Bar() - * { - * // Scan and print the contents of the directory '/usr/src/include' - * - * FastOS_DirectoryScan dirScan("/usr/src/include"); - * int numEntries = 0; - * - * while(dirScan.ReadNext()) - * { - * const char *name = dirScan.GetName(); - * bool isDirectory = dirScan.IsDirectory(); - * bool isRegular = dirScan.IsRegular(); - * - * printf("%-30s %s\n", name, - * isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); - * - * numEntries++; - * } - * - * printf("The directory contained %d entries.\n", numEntries); - * } - * @endcode - */ -class FastOS_DirectoryScanInterface -{ -protected: - std::string _searchPath; - -public: - FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&) = delete; - FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&) = delete; - - /** - * Constructor. - * - * @param path Path of the directory to be scanned. The path string - * is copied internally. - */ - FastOS_DirectoryScanInterface(const char *path); - - /** - * Destructor. - * - * Frees operating system resources related to the directory scan. - */ - virtual ~FastOS_DirectoryScanInterface(); - - /** - * Read the next entry in the directory scan. Failure indicates - * that there are no more entries. If the call is successful, - * attributes for the entry can be read with @ref IsDirectory(), - * @ref IsRegular() and @ref GetName(). - * - * @return Boolean success/failure - */ - virtual bool ReadNext() = 0; - - /** - * After a successful @ref ReadNext() this method is used to - * determine if the entry is a directory entry or not. Calling this - * method after an unsuccessful @ref ReadNext() or before - * @ref ReadNext() is called for the first time, yields undefined - * results. - * - * @return True if the entry is a directory, else false. - */ - virtual bool IsDirectory() = 0; - - - /** - * After a successful @ref ReadNext() this method is used to - * determine if the entry is a regular file entry or not. Calling - * this method after an unsuccessful @ref ReadNext() or before - * @ref ReadNext() is called for the first time, yields undefined - * results. - * - * @return True if the entry is a regular file, else false. - */ - virtual bool IsRegular() = 0; - - /** - * After a successful @ref ReadNext() this method is used to - * determine the name of the recently read directory entry. Calling - * this method after an unsuccessful @ref ReadNext() or before - * @ref ReadNext() is called for the first time, yields undefined - * results. - * - * @return A pointer to the recently read directory entry. - */ - virtual const char *GetName() = 0; -}; - #ifdef __linux__ #include <vespa/fastos/linux_file.h> typedef FastOS_Linux_File FASTOS_PREFIX(File); @@ -649,4 +550,3 @@ typedef FastOS_Linux_File FASTOS_PREFIX(File); #include <vespa/fastos/unix_file.h> typedef FastOS_UNIX_File FASTOS_PREFIX(File); #endif -typedef FastOS_UNIX_DirectoryScan FASTOS_PREFIX(DirectoryScan); diff --git a/vespalib/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp index b8ee005517a..3344250838c 100644 --- a/vespalib/src/vespa/fastos/linux_file.cpp +++ b/vespalib/src/vespa/fastos/linux_file.cpp @@ -11,6 +11,7 @@ #include "file.h" #include "file_rw_ops.h" #include <sstream> +#include <dirent.h> #include <unistd.h> #include <fcntl.h> #include <cstdio> diff --git a/vespalib/src/vespa/fastos/unix_file.cpp b/vespalib/src/vespa/fastos/unix_file.cpp index 6d10338aec1..b9fe46e920d 100644 --- a/vespalib/src/vespa/fastos/unix_file.cpp +++ b/vespalib/src/vespa/fastos/unix_file.cpp @@ -368,114 +368,3 @@ FastOS_UNIX_File::count_open_files() return 0; #endif } - -FastOS_UNIX_DirectoryScan::FastOS_UNIX_DirectoryScan(const char *searchPath) - : FastOS_DirectoryScanInterface(searchPath), - _statRun(false), - _isDirectory(false), - _isRegular(false), - _statName(nullptr), - _statFilenameP(nullptr), - _dir(nullptr), - _dp(nullptr) -{ - _dir = opendir(searchPath); - - const int minimumLength = 512 + 1; - const int defaultLength = 16384; - - int maxNameLength = FastOS_File::GetMaximumFilenameLength(searchPath); - int maxPathLength = FastOS_File::GetMaximumPathLength(searchPath); - int nameLength = maxNameLength + 1 + maxPathLength; - - if ((maxNameLength == -1) || - (maxPathLength == -1) || - (nameLength < minimumLength)) - { - nameLength = defaultLength; - } - - _statName = new char [nameLength + 1]; // Include null - - strcpy(_statName, searchPath); - strcat(_statName, "/"); - - _statFilenameP = &_statName[strlen(_statName)]; -} - - -FastOS_UNIX_DirectoryScan::~FastOS_UNIX_DirectoryScan() -{ - if (_dir != nullptr) { - closedir(_dir); - _dir = nullptr; - } - delete [] _statName; -} - - -bool -FastOS_UNIX_DirectoryScan::ReadNext() -{ - _statRun = false; - - if (_dir != nullptr) { - _dp = readdir(_dir); - return (_dp != nullptr); - } - - return false; -} - - -void -FastOS_UNIX_DirectoryScan::DoStat() -{ - struct stat stbuf{}; - - assert(_dp != nullptr); - - strcpy(_statFilenameP, _dp->d_name); - - if (lstat(_statName, &stbuf) == 0) { - _isRegular = S_ISREG(stbuf.st_mode); - _isDirectory = S_ISDIR(stbuf.st_mode); - } else { - printf("lstat failed for [%s]\n", _dp->d_name); - _isRegular = false; - _isDirectory = false; - } - - _statRun = true; -} - - -bool -FastOS_UNIX_DirectoryScan::IsDirectory() -{ - if (!_statRun) { - DoStat(); - } - - return _isDirectory; -} - - -bool -FastOS_UNIX_DirectoryScan::IsRegular() -{ - if (!_statRun) { - DoStat(); - } - - return _isRegular; -} - - -const char * -FastOS_UNIX_DirectoryScan::GetName() -{ - assert(_dp != nullptr); - - return static_cast<const char *>(_dp->d_name); -} diff --git a/vespalib/src/vespa/fastos/unix_file.h b/vespalib/src/vespa/fastos/unix_file.h index dad75dc561f..81e5de901a3 100644 --- a/vespalib/src/vespa/fastos/unix_file.h +++ b/vespalib/src/vespa/fastos/unix_file.h @@ -4,7 +4,7 @@ * @author Oivind H. Danielsen * @date Creation date: 2000-01-18 * @file -* Class definitions for FastOS_UNIX_File and FastOS_UNIX_DirectoryScan. +* Class definitions for FastOS_UNIX_File *****************************************************************************/ #pragma once @@ -83,32 +83,3 @@ public: static int64_t GetFreeDiskSpace (const char *path); static int count_open_files(); }; - -#include <dirent.h> -/** - * This is the generic UNIX implementation of @ref FastOS_DirectoryScan. - */ -class FastOS_UNIX_DirectoryScan : public FastOS_DirectoryScanInterface -{ -private: - bool _statRun; - bool _isDirectory; - bool _isRegular; - char *_statName; - char *_statFilenameP; - - void DoStat(); - -protected: - DIR *_dir; - struct dirent *_dp; - -public: - FastOS_UNIX_DirectoryScan(const char *searchPath); - ~FastOS_UNIX_DirectoryScan(); - - bool ReadNext() override; - bool IsDirectory() override; - bool IsRegular() override; - const char *GetName() override; -}; |