From 91596d876fbb5da8c142f160708dc99086e2fddf Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 11 Jun 2021 10:18:31 +0200 Subject: Log load balancer state transitions --- .../provision/maintenance/LoadBalancerExpirer.java | 4 +- .../persistence/CuratorDatabaseClient.java | 17 ++++- .../provisioning/LoadBalancerProvisioner.java | 87 ++++++++++------------ 3 files changed, 57 insertions(+), 51 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index ac9d8d6671a..c3d6f5c42b8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -65,7 +65,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { Instant now = nodeRepository().clock().instant(); Instant expiry = now.minus(reservedExpiry); patchLoadBalancers(lb -> lb.state() == State.reserved && lb.changedAt().isBefore(expiry), - lb -> db.writeLoadBalancer(lb.with(State.inactive, now))); + lb -> db.writeLoadBalancer(lb.with(State.inactive, now), lb.state())); } /** Deprovision inactive load balancers that have expired */ @@ -114,7 +114,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { attempts.add(1); LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals)); service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals), true); - db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals)))); + db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals))), lb.state()); } catch (Exception e) { failed.add(lb.id()); lastException.set(e); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 364af6308a1..0205cc6c818 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -482,18 +482,29 @@ public class CuratorDatabaseClient { return read(loadBalancerPath(id), LoadBalancerSerializer::fromJson); } - public void writeLoadBalancer(LoadBalancer loadBalancer) { + public void writeLoadBalancer(LoadBalancer loadBalancer, LoadBalancer.State fromState) { NestedTransaction transaction = new NestedTransaction(); - writeLoadBalancers(List.of(loadBalancer), transaction); + writeLoadBalancers(List.of(loadBalancer), fromState, transaction); transaction.commit(); } - public void writeLoadBalancers(Collection loadBalancers, NestedTransaction transaction) { + public void writeLoadBalancers(Collection loadBalancers, LoadBalancer.State fromState, NestedTransaction transaction) { CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); loadBalancers.forEach(loadBalancer -> { curatorTransaction.add(createOrSet(loadBalancerPath(loadBalancer.id()), LoadBalancerSerializer.toJson(loadBalancer))); }); + transaction.onCommitted(() -> { + for (var lb : loadBalancers) { + if (lb.state() == fromState) continue; + if (fromState == null) { + log.log(Level.INFO, () -> "Creating " + lb.id() + " in " + lb.state()); + } else { + log.log(Level.INFO, () -> "Moving " + lb.id() + " from " + fromState + + " to " + lb.state()); + } + } + }); } public void removeLoadBalancer(LoadBalancerId loadBalancer) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index c114aa58a05..71884829b62 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; @@ -61,7 +60,7 @@ public class LoadBalancerProvisioner { for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { var loadBalancer = db.readLoadBalancer(id); - loadBalancer.ifPresent(db::writeLoadBalancer); + loadBalancer.ifPresent(lb -> db.writeLoadBalancer(lb, lb.state())); } } } @@ -81,11 +80,9 @@ public class LoadBalancerProvisioner { if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lock(application)) { ClusterSpec.Id clusterId = effectiveId(cluster); - NodeList nodes = nodesOf(clusterId, application); LoadBalancerId loadBalancerId = requireNonClashing(new LoadBalancerId(application, clusterId)); - ApplicationTransaction transaction = new ApplicationTransaction(new ProvisionLock(application, lock), new NestedTransaction()); - provision(transaction, loadBalancerId, nodes, false); - transaction.nested().commit(); + NodeList nodes = nodesOf(clusterId, application); + prepare(loadBalancerId, nodes); } } @@ -100,14 +97,15 @@ public class LoadBalancerProvisioner { * Calling this when no load balancer has been prepared for given cluster is a no-op. */ public void activate(Set clusters, ApplicationTransaction transaction) { + Set activatingClusters = clusters.stream() + .map(LoadBalancerProvisioner::effectiveId) + .collect(Collectors.toSet()); for (var cluster : loadBalancedClustersOf(transaction.application()).entrySet()) { - // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(transaction, cluster.getKey(), cluster.getValue()); + if (!activatingClusters.contains(cluster.getKey())) continue; + activate(transaction, cluster.getKey(), cluster.getValue()); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), clusters.stream() - .map(LoadBalancerProvisioner::effectiveId) - .collect(Collectors.toSet())); + var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), activatingClusters); deactivate(surplusLoadBalancers, transaction.nested()); } @@ -140,7 +138,7 @@ public class LoadBalancerProvisioner { var deactivatedLoadBalancers = loadBalancers.stream() .map(lb -> lb.with(LoadBalancer.State.inactive, now)) .collect(Collectors.toList()); - db.writeLoadBalancers(deactivatedLoadBalancers, transaction); + db.writeLoadBalancers(deactivatedLoadBalancers, LoadBalancer.State.active, transaction); } /** Find all load balancer IDs owned by given tenant and application */ @@ -165,52 +163,41 @@ public class LoadBalancerProvisioner { return loadBalancerId; } - /** Idempotently provision a load balancer for given application and cluster */ - private void provision(ApplicationTransaction transaction, LoadBalancerId id, NodeList nodes, boolean activate) { + private void prepare(LoadBalancerId id, NodeList nodes) { Instant now = nodeRepository.clock().instant(); Optional loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty() && activate) return; // Nothing to activate as this load balancer was never prepared - - Set reals = realsOf(nodes); - Optional instance = provisionInstance(id, reals, loadBalancer); + Optional instance = provisionInstance(id, nodes, loadBalancer); LoadBalancer newLoadBalancer; + LoadBalancer.State fromState = null; if (loadBalancer.isEmpty()) { newLoadBalancer = new LoadBalancer(id, instance, LoadBalancer.State.reserved, now); } else { - LoadBalancer.State state = activate && instance.isPresent() - ? LoadBalancer.State.active - : loadBalancer.get().state(); - newLoadBalancer = loadBalancer.get().with(instance).with(state, now); - if (loadBalancer.get().state() != newLoadBalancer.state()) { - log.log(Level.INFO, () -> "Moving " + newLoadBalancer.id() + " from " + loadBalancer.get().state() + - " to " + newLoadBalancer.state()); - } - } - - if (activate) { - db.writeLoadBalancers(List.of(newLoadBalancer), transaction.nested()); - } else { - // Always store load balancer so that LoadBalancerExpirer can expire partially provisioned load balancers - db.writeLoadBalancer(newLoadBalancer); - } - - // Signal that load balancer is not ready yet - if (instance.isEmpty()) { - throw new LoadBalancerServiceException("Could not (re)configure " + id + ", targeting: " + - reals + ". The operation will be retried on next deployment", - null); + newLoadBalancer = loadBalancer.get().with(instance); + fromState = newLoadBalancer.state(); } + // Always store load balancer so that LoadBalancerExpirer can expire partially provisioned load balancers + db.writeLoadBalancer(newLoadBalancer, fromState); + requireInstance(id, instance); } - private void provision(ApplicationTransaction transaction, ClusterSpec.Id clusterId, NodeList nodes) { - provision(transaction, new LoadBalancerId(transaction.application(), clusterId), nodes, true); + private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, NodeList nodes) { + Instant now = nodeRepository.clock().instant(); + LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster); + Optional loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not active load balancer that was never prepared: " + id); + + Optional instance = provisionInstance(id, nodes, loadBalancer); + LoadBalancer.State state = instance.isPresent() ? LoadBalancer.State.active : loadBalancer.get().state(); + LoadBalancer newLoadBalancer = loadBalancer.get().with(instance).with(state, now); + db.writeLoadBalancers(List.of(newLoadBalancer), loadBalancer.get().state(), transaction.nested()); + requireInstance(id, instance); } /** Provision or reconfigure a load balancer instance, if necessary */ - private Optional provisionInstance(LoadBalancerId id, Set reals, - Optional currentLoadBalancer) { + private Optional provisionInstance(LoadBalancerId id, NodeList nodes, Optional currentLoadBalancer) { + Set reals = realsOf(nodes); if (hasReals(currentLoadBalancer, reals)) return currentLoadBalancer.get().instance(); - log.log(Level.INFO, () -> "Creating " + id + ", targeting: " + reals); + log.log(Level.INFO, () -> "Provisioning instance for " + id + ", targeting: " + reals); try { return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals), allowEmptyReals(currentLoadBalancer))); @@ -241,7 +228,7 @@ public class LoadBalancerProvisioner { /** Returns real servers for given nodes */ private Set realsOf(NodeList nodes) { - var reals = new LinkedHashSet(); + Set reals = new LinkedHashSet(); for (var node : nodes) { for (var ip : reachableIpAddresses(node)) { reals.add(new Real(HostName.from(node.hostname()), ip)); @@ -289,6 +276,14 @@ public class LoadBalancerProvisioner { return reachable; } + private static void requireInstance(LoadBalancerId id, Optional instance) { + if (instance.isEmpty()) { + // Signal that load balancer is not ready yet + throw new LoadBalancerServiceException("Could not (re)configure " + id + ". The operation will be retried on next deployment", + null); + } + } + private static ClusterSpec.Id effectiveId(ClusterSpec cluster) { return cluster.combinedId().orElse(cluster.id()); } -- cgit v1.2.3 From 1032d77abd632500a7128c9bbd3a8e494d2dc287 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 11 Jun 2021 15:37:16 +0200 Subject: Test that service supports node/cluster type when activating --- .../vespa/hosted/provision/lb/LoadBalancerService.java | 5 +---- .../vespa/hosted/provision/lb/LoadBalancerServiceMock.java | 14 ++++++++++++++ .../provision/lb/PassthroughLoadBalancerService.java | 6 ++++++ .../hosted/provision/lb/SharedLoadBalancerService.java | 2 +- .../provision/provisioning/LoadBalancerProvisioner.java | 5 ++++- .../provisioning/LoadBalancerProvisionerTest.java | 7 +++++++ 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index fba0993f2f9..40f9b330634 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -29,10 +29,7 @@ public interface LoadBalancerService { Protocol protocol(); /** Returns whether load balancers created by this service can forward traffic to given node and cluster type */ - default boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { - return (nodeType == NodeType.tenant && clusterType.isContainer()) || - nodeType.isConfigServerLike(); - } + boolean supports(NodeType nodeType, ClusterSpec.Type clusterType); /** Load balancer protocols */ enum Protocol { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index b912087da46..f752cbc4349 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; import java.util.Collections; import java.util.HashMap; @@ -18,6 +19,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { private final Map instances = new HashMap<>(); private boolean throwOnCreate = false; + private boolean supportsProvisioning = true; public Map instances() { return Collections.unmodifiableMap(instances); @@ -28,6 +30,18 @@ public class LoadBalancerServiceMock implements LoadBalancerService { return this; } + public LoadBalancerServiceMock supportsProvisioning(boolean supportsProvisioning) { + this.supportsProvisioning = supportsProvisioning; + return this; + } + + @Override + public boolean supports(NodeType nodeType, ClusterSpec.Type clusterType) { + if (!supportsProvisioning) return false; + return (nodeType == NodeType.tenant && clusterType.isContainer()) || + nodeType.isConfigServerLike(); + } + @Override public Protocol protocol() { return Protocol.ipv4; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java index 7667672e470..9a6a65eca69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeType; import java.util.Comparator; import java.util.Optional; @@ -35,4 +36,9 @@ public class PassthroughLoadBalancerService implements LoadBalancerService { return Protocol.ipv4; } + @Override + public boolean supports(NodeType nodeType, ClusterSpec.Type clusterType) { + return true; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 33a3c138d70..e17e5a5a449 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -66,7 +66,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { + public boolean supports(NodeType nodeType, ClusterSpec.Type clusterType) { // Shared routing layer only supports routing to tenant nodes return nodeType == NodeType.tenant && clusterType.isContainer(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 71884829b62..b6600574f94 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -76,7 +76,7 @@ public class LoadBalancerProvisioner { * Calling this for irrelevant node or cluster types is a no-op. */ public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { - if (!service.canForwardTo(requestedNodes.type(), cluster.type())) return; // Nothing to provision for this node and cluster type + if (!service.supports(requestedNodes.type(), cluster.type())) return; // Nothing to provision for this node and cluster type if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lock(application)) { ClusterSpec.Id clusterId = effectiveId(cluster); @@ -102,6 +102,9 @@ public class LoadBalancerProvisioner { .collect(Collectors.toSet()); for (var cluster : loadBalancedClustersOf(transaction.application()).entrySet()) { if (!activatingClusters.contains(cluster.getKey())) continue; + + Node clusterNode = cluster.getValue().first().get(); + if (!service.supports(clusterNode.type(), clusterNode.allocation().get().membership().cluster().type())) continue; activate(transaction, cluster.getKey(), cluster.getValue()); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index bdc3bdfd816..16fe5ef241a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -300,6 +300,13 @@ public class LoadBalancerProvisionerTest { assertTrue("Load balancer has instance", loadBalancers.get(0).instance().isPresent()); } + @Test + public void provisioning_load_balancer_for_unsupported_cluster_fails_gracefully() { + tester.loadBalancerService().supportsProvisioning(false); + tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")))); + assertTrue("No load balancer provisioned", tester.nodeRepository().loadBalancers().list(app1).asList().isEmpty()); + } + private void dirtyNodesOf(ApplicationId application) { tester.nodeRepository().nodes().deallocate(tester.nodeRepository().nodes().list().owner(application).asList(), Agent.system, this.getClass().getSimpleName()); } -- cgit v1.2.3 From 6c9fabbdec12ed06aac9a401f85c98b00d1a1a7c Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 14 Jun 2021 10:03:01 +0000 Subject: add "connectivity" option to vespa-sentinel-cmd --- configd/src/apps/cmd/main.cpp | 98 +++++++++++++++++++++++++++++++------------ 1 file changed, 71 insertions(+), 27 deletions(-) diff --git a/configd/src/apps/cmd/main.cpp b/configd/src/apps/cmd/main.cpp index 33b4aa8111d..7e5fbba1465 100644 --- a/configd/src/apps/cmd/main.cpp +++ b/configd/src/apps/cmd/main.cpp @@ -13,6 +13,24 @@ #include LOG_SETUP("vespa-sentinel-cmd"); +namespace { +struct Method { + const char * name; + const char * rpcMethod; + bool noArgNeeded; + bool needsTimeoutArg; +}; +const Method methods[] = { + { "list", "sentinel.ls", true, false }, + { "restart", "sentinel.service.restart", false, false }, + { "start", "sentinel.service.start", false, false }, + { "stop", "sentinel.service.stop", false, false }, + { "connectivity", "sentinel.report.connectivity", true, true } +}; + +} + + class Cmd { private: @@ -22,7 +40,7 @@ private: public: Cmd() : _server(), _target(nullptr) {} ~Cmd(); - int run(const char *cmd, const char *arg); + int run(const Method &cmd, const char *arg); void initRPC(const char *spec); void finiRPC(); }; @@ -41,6 +59,7 @@ void usage() fprintf(stderr, " restart {service}\n"); fprintf(stderr, " start {service}\n"); fprintf(stderr, " stop {service}\n"); + fprintf(stderr, " connectivity [milliseconds]\n"); } void @@ -63,7 +82,7 @@ Cmd::finiRPC() int -Cmd::run(const char *cmd, const char *arg) +Cmd::run(const Method &cmd, const char *arg) { int retval = 0; try { @@ -74,33 +93,61 @@ Cmd::run(const char *cmd, const char *arg) return 2; } FRT_RPCRequest *req = _server->supervisor().AllocRPCRequest(); - req->SetMethodName(cmd); + req->SetMethodName(cmd.rpcMethod); - if (arg) { + if (cmd.needsTimeoutArg) { + int timeoutMs = 5000; + if (arg) { + timeoutMs = atoi(arg); + } + req->GetParams()->AddInt32(timeoutMs); + } else if (arg) { // one param req->GetParams()->AddString(arg); } - _target->InvokeSync(req, 5.0); + _target->InvokeSync(req, 10.0); if (req->IsError()) { fprintf(stderr, "vespa-sentinel-cmd '%s' error %d: %s\n", - cmd, req->GetErrorCode(), req->GetErrorMessage()); + cmd.name, req->GetErrorCode(), req->GetErrorMessage()); retval = 1; } else { FRT_Values &answer = *(req->GetReturn()); const char *atypes = answer.GetTypeString(); - fprintf(stderr, "vespa-sentinel-cmd '%s' OK.\n", cmd); - uint32_t idx = 0; - while (atypes != nullptr && *atypes != '\0') { - switch (*atypes) { - case 's': + fprintf(stderr, "vespa-sentinel-cmd '%s' OK.\n", cmd.name); + if (atypes && (strcmp(atypes, "SS") == 0)) { + uint32_t numHosts = answer[0]._string_array._len; + uint32_t numStats = answer[1]._string_array._len; + FRT_StringValue *hosts = answer[0]._string_array._pt; + FRT_StringValue *stats = answer[1]._string_array._pt; + uint32_t ml = 0; + uint32_t j; + for (j = 0; j < numHosts; ++j) { + uint32_t hl = strlen(hosts[j]._str); + if (hl > ml) ml = hl; + } + for (j = 0; j < numHosts && j < numStats; ++j) { + printf("%-*s -> %s\n", ml, hosts[j]._str, stats[j]._str); + } + for (; j < numHosts; ++j) { + printf("Extra host: %s\n", hosts[j]._str); + } + for (; j < numStats; ++j) { + printf("Extra stat: %s\n", stats[j]._str); + } + } else { + uint32_t idx = 0; + while (atypes != nullptr && *atypes != '\0') { + switch (*atypes) { + case 's': printf("%s\n", answer[idx]._string._str); break; - default: + default: printf("BAD: unknown type %c\n", *atypes); - } - ++atypes; + } + ++atypes; ++idx; + } } } req->SubRef(); @@ -108,19 +155,15 @@ Cmd::run(const char *cmd, const char *arg) return retval; } -const char * +const Method * parseCmd(const char *arg) { - if (strcmp(arg, "list") == 0) { - return "sentinel.ls"; - } else if (strcmp(arg, "restart") == 0) { - return "sentinel.service.restart"; - } else if (strcmp(arg, "start") == 0) { - return "sentinel.service.start"; - } else if (strcmp(arg, "stop") == 0) { - return "sentinel.service.stop"; + for (const auto & method : methods) { + if (strcmp(arg, method.name) == 0) { + return &method; + } } - return 0; + return nullptr; } void hookSignals() { @@ -131,14 +174,15 @@ void hookSignals() { int main(int argc, char** argv) { int retval = 1; - const char *cmd = 0; + const Method *cmd = nullptr; if (argc > 1) { cmd = parseCmd(argv[1]); } - if (cmd) { + const char *extraArg = (argc > 2 ? argv[2] : nullptr); + if (cmd && (extraArg || cmd->noArgNeeded)) { hookSignals(); Cmd runner; - retval = runner.run(cmd, argc > 2 ? argv[2] : 0); + retval = runner.run(*cmd, extraArg); } else { usage(); } -- cgit v1.2.3 From 8fdcba59a3a588ed40b9ff98daac52f08dfc01d5 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 14 Jun 2021 11:38:43 +0000 Subject: Use a list instead of a set to make building faster. Then sort and uniq before applying the list. --- .../searchlib/attribute/enum_store_dictionary.cpp | 28 +++++++++--------- .../src/vespa/searchlib/attribute/enumstore.h | 10 +++---- .../src/vespa/searchlib/attribute/enumstore.hpp | 16 ++++++++--- .../src/vespa/searchlib/attribute/i_enum_store.h | 12 ++------ .../searchlib/attribute/i_enum_store_dictionary.h | 2 +- .../searchlib/attribute/singleenumattribute.hpp | 6 ++-- .../tests/datastore/datastore/datastore_test.cpp | 33 ++++++++++++++++++++++ vespalib/src/vespa/vespalib/datastore/entryref.h | 1 + 8 files changed, 70 insertions(+), 38 deletions(-) diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index d867ae9f211..1b21f3a7b6e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -1,13 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "enum_store_dictionary.h" -#include "enumstore.h" #include -#include #include -#include -#include -#include #include #include #include @@ -28,9 +23,6 @@ void EnumStoreDictionary::remove_unused_values(const IndexSet& unused, const vespalib::datastore::EntryComparator& cmp) { - if (unused.empty()) { - return; - } for (const auto& ref : unused) { this->remove(cmp, ref); } @@ -58,7 +50,9 @@ EnumStoreDictionary::free_unused_values(const _enumStore.free_value_if_unused(iter.getKey(), unused); } } else { - this->_hash_dict.foreach_key([this, &unused](EntryRef ref) { _enumStore.free_value_if_unused(ref, unused); }); + this->_hash_dict.foreach_key([this, &unused](EntryRef ref) { + _enumStore.free_value_if_unused(ref, unused); + }); } remove_unused_values(unused, cmp); } @@ -66,11 +60,17 @@ EnumStoreDictionary::free_unused_values(const template void EnumStoreDictionary::free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) + const vespalib::datastore::EntryComparator& cmp) { IndexSet unused; + + EntryRef prev; for (const auto& index : to_remove) { - _enumStore.free_value_if_unused(index, unused); + assert(prev <= index); + if (index != prev) { + _enumStore.free_value_if_unused(index, unused); + prev = index; + } } remove_unused_values(unused, cmp); } @@ -96,8 +96,7 @@ EnumStoreDictionary::remove(const EntryCompar template bool -EnumStoreDictionary::find_index(const vespalib::datastore::EntryComparator& cmp, - Index& idx) const +EnumStoreDictionary::find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const { if constexpr (has_hash_dictionary) { auto find_result = this->_hash_dict.find(cmp, EntryRef()); @@ -118,8 +117,7 @@ EnumStoreDictionary::find_index(const vespali template bool -EnumStoreDictionary::find_frozen_index(const vespalib::datastore::EntryComparator& cmp, - Index& idx) const +EnumStoreDictionary::find_frozen_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const { if constexpr (has_hash_dictionary) { auto find_result = this->_hash_dict.find(cmp, EntryRef()); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 326e0916039..59d77ea0558 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -63,7 +63,7 @@ private: EnumStoreT(const EnumStoreT & rhs) = delete; EnumStoreT & operator=(const EnumStoreT & rhs) = delete; - void free_value_if_unused(Index idx, IndexSet &unused) override; + void free_value_if_unused(Index idx, IndexList &unused) override; const vespalib::datastore::UniqueStoreEntryBase& get_entry_base(Index idx) const { return _store.get_allocator().get_wrapped(idx); @@ -153,7 +153,7 @@ public: class BatchUpdater { private: EnumStoreType& _store; - IndexSet _possibly_unused; + IndexList _possibly_unused; public: BatchUpdater(EnumStoreType& store) @@ -168,11 +168,11 @@ public: auto& entry = _store.get_entry_base(idx); entry.dec_ref_count(); if (entry.get_ref_count() == 0) { - _possibly_unused.insert(idx); + _possibly_unused.push_back(idx); } } void commit() { - _store.free_unused_values(_possibly_unused); + _store.free_unused_values(std::move(_possibly_unused)); } }; @@ -198,7 +198,7 @@ public: Index insert(EntryType value); bool find_index(EntryType value, Index& idx) const; void free_unused_values() override; - void free_unused_values(const IndexSet& to_remove); + void free_unused_values(IndexList to_remove); vespalib::MemoryUsage update_stat() override; std::unique_ptr consider_compact_values(const CompactionStrategy& compaction_strategy) override; std::unique_ptr compact_worst_values(bool compact_memory, bool compact_address_space) override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 9885613f4e3..771da8ffa01 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -30,11 +30,11 @@ make_enum_store_dictionary(IEnumStore &store, bool has_postings, const search::D std::unique_ptr folded_compare); template -void EnumStoreT::free_value_if_unused(Index idx, IndexSet& unused) +void EnumStoreT::free_value_if_unused(Index idx, IndexList& unused) { const auto& entry = get_entry_base(idx); if (entry.get_ref_count() == 0) { - unused.insert(idx); + unused.push_back(idx); _store.get_allocator().hold(idx); } } @@ -140,7 +140,7 @@ EnumStoreT::BatchUpdater::insert(EntryType value) auto cmp = _store.make_comparator(value); auto result = _store._dict->add(cmp, [this, &value]() -> EntryRef { return _store._store.get_allocator().allocate(value); }); if (result.inserted()) { - _possibly_unused.insert(result.ref()); + _possibly_unused.push_back(result.ref()); } return result.ref(); } @@ -191,8 +191,16 @@ EnumStoreT::free_unused_values() template void -EnumStoreT::free_unused_values(const IndexSet& to_remove) +EnumStoreT::free_unused_values(IndexList to_remove) { + struct CompareEnumIndex { + using Index = IEnumStore::Index; + + bool operator()(const Index &lhs, const Index &rhs) const { + return lhs.ref() < rhs.ref(); + } + }; + std::sort(to_remove.begin(), to_remove.end(), CompareEnumIndex()); _dict->free_unused_values(to_remove, get_comparator()); } diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 6d714ec25ba..716609764f4 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -40,22 +40,14 @@ public: using EnumIndexRemapper = vespalib::datastore::UniqueStoreRemapper; using Enumerator = vespalib::datastore::UniqueStoreEnumerator; - struct CompareEnumIndex { - using Index = IEnumStore::Index; - - bool operator()(const Index &lhs, const Index &rhs) const { - return lhs.ref() < rhs.ref(); - } - }; - - using IndexSet = std::set; + using IndexList = std::vector; virtual ~IEnumStore() = default; virtual void write_value(BufferWriter& writer, Index idx) const = 0; virtual ssize_t load_unique_values(const void* src, size_t available, IndexVector& idx) = 0; virtual void set_ref_count(Index idx, uint32_t ref_count) = 0; - virtual void free_value_if_unused(Index idx, IndexSet& unused) = 0; + virtual void free_value_if_unused(Index idx, IndexList& unused) = 0; virtual void free_unused_values() = 0; virtual bool is_folded_change(Index idx1, Index idx2) const = 0; virtual IEnumStoreDictionary& get_dictionary() = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index f816177b06c..9d72369f245 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -31,7 +31,7 @@ public: using EntryRef = vespalib::datastore::EntryRef; using EnumVector = IEnumStore::EnumVector; using Index = IEnumStore::Index; - using IndexSet = IEnumStore::IndexSet; + using IndexSet = IEnumStore::IndexList; using IndexVector = IEnumStore::IndexVector; using generation_t = vespalib::GenerationHandler::generation_t; diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index bf75400b157..a9a94afb763 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -312,9 +312,9 @@ SingleValueEnumAttribute::onShrinkLidSpace() uint32_t default_value_ref_count = this->_enumStore.get_ref_count(default_value_ref); assert(default_value_ref_count >= shrink_docs); this->_enumStore.set_ref_count(default_value_ref, default_value_ref_count - shrink_docs); - IEnumStore::IndexSet possibly_unused; - possibly_unused.insert(default_value_ref); - this->_enumStore.free_unused_values(possibly_unused); + IEnumStore::IndexList possibly_unused; + possibly_unused.push_back(default_value_ref); + this->_enumStore.free_unused_values(std::move(possibly_unused)); } _enumIndices.shrink(committedDocIdLimit); this->setNumDocs(committedDocIdLimit); diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index 11b4f5e6631..90281acb0d3 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -141,6 +141,38 @@ assertMemStats(const DataStoreBase::MemStats &exp, EXPECT_EQ(exp._holdBuffers, act._holdBuffers); } +TEST(DataStoreTest, require_that_invalid_entry_ref_can_be_ordered) { + EntryRef inValid; + EntryRef a(1); + EXPECT_EQ(inValid, inValid); + EXPECT_EQ(a, a); + EXPECT_NE(inValid, a); + EXPECT_NE(a, inValid); + EXPECT_LT(inValid, a); + EXPECT_LE(inValid, a); +} + +TEST(DataStoreTest, require_that_entry_ref_can_be_ordered) { + EntryRef a(1); + EntryRef b(2); + EntryRef c(3); + EXPECT_EQ(a, a); + EXPECT_EQ(b, b); + EXPECT_EQ(c, c); + EXPECT_NE(a, b); + EXPECT_NE(a, c); + EXPECT_NE(b, c); + EXPECT_LT(a, b); + EXPECT_LT(b, c); + EXPECT_LT(a, c); + EXPECT_LE(a, a); + EXPECT_LE(b, b); + EXPECT_LE(c, c); + EXPECT_LE(a, b); + EXPECT_LE(b, c); + EXPECT_LE(a, c); +} + TEST(DataStoreTest, require_that_entry_ref_is_working) { using MyRefType = EntryRefT<22>; @@ -643,6 +675,7 @@ TEST(DataStoreTest, control_static_sizes) { EXPECT_EQ(0, bs.size()); } + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.h b/vespalib/src/vespa/vespalib/datastore/entryref.h index 046d9089580..01f473fcf17 100644 --- a/vespalib/src/vespa/vespalib/datastore/entryref.h +++ b/vespalib/src/vespa/vespalib/datastore/entryref.h @@ -21,6 +21,7 @@ public: bool operator==(const EntryRef &rhs) const noexcept { return _ref == rhs._ref; } bool operator!=(const EntryRef &rhs) const noexcept { return _ref != rhs._ref; } bool operator <(const EntryRef &rhs) const noexcept { return _ref < rhs._ref; } + bool operator <=(const EntryRef &rhs) const noexcept { return _ref <= rhs._ref; } }; /** -- cgit v1.2.3 From 86be1befadd23afb27b76d2f4f7b4d4d0fb2b1fe Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Mon, 14 Jun 2021 14:26:38 +0200 Subject: Aggregate distributor metrics when adding to snapshot. --- .../vespa/storage/distributor/distributor_total_metrics.cpp | 11 +++++++++++ .../src/vespa/storage/distributor/distributor_total_metrics.h | 1 + 2 files changed, 12 insertions(+) diff --git a/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp b/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp index 543712cc4d2..fff996958b3 100644 --- a/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp +++ b/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp @@ -27,6 +27,17 @@ DistributorTotalMetrics::aggregate() } } +void +DistributorTotalMetrics::addToSnapshot(Metric& m, std::vector &ownerList) const +{ + DistributorMetricSet total; + _bucket_db_updater_metrics.addToPart(total); + for (auto &stripe_metrics : _stripes_metrics) { + stripe_metrics->addToPart(total); + } + total.addToSnapshot(m, ownerList); +} + void DistributorTotalMetrics::reset() { diff --git a/storage/src/vespa/storage/distributor/distributor_total_metrics.h b/storage/src/vespa/storage/distributor/distributor_total_metrics.h index 14116af3d3b..7b4f705a6fe 100644 --- a/storage/src/vespa/storage/distributor/distributor_total_metrics.h +++ b/storage/src/vespa/storage/distributor/distributor_total_metrics.h @@ -19,6 +19,7 @@ public: explicit DistributorTotalMetrics(uint32_t num_distributor_stripes); ~DistributorTotalMetrics() override; void aggregate(); + void addToSnapshot(Metric& m, std::vector &ownerList) const override; void reset() override; DistributorMetricSet& stripe(uint32_t stripe_index) { return *_stripes_metrics[stripe_index]; } DistributorMetricSet& bucket_db_updater_metrics() { return _bucket_db_updater_metrics; } -- cgit v1.2.3 From 98c2bc9669c19c92a749d34990745d31b5e3504f Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Mon, 14 Jun 2021 14:30:52 +0200 Subject: Avoid triggering disk encryption on proxy hosts for now --- .../com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java index 80f74a011c0..6d88e43630a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java @@ -48,6 +48,8 @@ public class HostEncrypter extends NodeRepositoryMaintainer { NodeList allNodes = nodeRepository().nodes().list(); for (var nodeType : NodeType.values()) { if (!nodeType.isHost()) continue; + // TODO: Require a minimum number of proxies in Orchestrator. For now skip proxy hosts. + if (nodeType == NodeType.proxyhost) continue; if (upgradingVespa(allNodes, nodeType)) continue; unencryptedHosts(allNodes, nodeType).forEach(host -> encrypt(host, now)); } -- cgit v1.2.3 From ffd882eb021758a293b2ded2451d4981de8801f6 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Mon, 14 Jun 2021 14:46:05 +0200 Subject: Factor out common code to private helper member function. --- .../storage/distributor/distributor_total_metrics.cpp | 19 +++++++++++-------- .../storage/distributor/distributor_total_metrics.h | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp b/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp index fff996958b3..510b1df2ff3 100644 --- a/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp +++ b/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp @@ -18,23 +18,26 @@ DistributorTotalMetrics::DistributorTotalMetrics(uint32_t num_distributor_stripe DistributorTotalMetrics::~DistributorTotalMetrics() = default; void -DistributorTotalMetrics::aggregate() +DistributorTotalMetrics::aggregate_helper(DistributorMetricSet &total) const { - DistributorMetricSet::reset(); - _bucket_db_updater_metrics.addToPart(*this); + _bucket_db_updater_metrics.addToPart(total); for (auto &stripe_metrics : _stripes_metrics) { - stripe_metrics->addToPart(*this); + stripe_metrics->addToPart(total); } } +void +DistributorTotalMetrics::aggregate() +{ + DistributorMetricSet::reset(); + aggregate_helper(*this); +} + void DistributorTotalMetrics::addToSnapshot(Metric& m, std::vector &ownerList) const { DistributorMetricSet total; - _bucket_db_updater_metrics.addToPart(total); - for (auto &stripe_metrics : _stripes_metrics) { - stripe_metrics->addToPart(total); - } + aggregate_helper(total); total.addToSnapshot(m, ownerList); } diff --git a/storage/src/vespa/storage/distributor/distributor_total_metrics.h b/storage/src/vespa/storage/distributor/distributor_total_metrics.h index 7b4f705a6fe..f0457fe64c3 100644 --- a/storage/src/vespa/storage/distributor/distributor_total_metrics.h +++ b/storage/src/vespa/storage/distributor/distributor_total_metrics.h @@ -15,6 +15,7 @@ class DistributorTotalMetrics : public DistributorMetricSet { std::vector> _stripes_metrics; DistributorMetricSet _bucket_db_updater_metrics; + void aggregate_helper(DistributorMetricSet &total) const; public: explicit DistributorTotalMetrics(uint32_t num_distributor_stripes); ~DistributorTotalMetrics() override; -- cgit v1.2.3 From 5f2365e439a4e15b901397a1fcd2c0e79843fd43 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 14 Jun 2021 14:55:16 +0200 Subject: Add Javadoc --- .../main/java/ai/vespa/feed/client/DocumentId.java | 2 ++ .../main/java/ai/vespa/feed/client/FeedClient.java | 2 ++ .../ai/vespa/feed/client/FeedClientBuilder.java | 26 ++++++++++++++++++++++ .../java/ai/vespa/feed/client/FeedException.java | 2 ++ .../ai/vespa/feed/client/JsonParseException.java | 2 ++ .../ai/vespa/feed/client/OperationParameters.java | 2 ++ .../src/main/java/ai/vespa/feed/client/Result.java | 2 ++ 7 files changed, 38 insertions(+) diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/DocumentId.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/DocumentId.java index 21513a5dac2..39fc9fb28e0 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/DocumentId.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/DocumentId.java @@ -8,6 +8,8 @@ import java.util.OptionalLong; import static java.util.Objects.requireNonNull; /** + * Represents a Vespa document id + * * @author jonmv */ public class DocumentId { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java index 39d343515fe..250809a48b9 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java @@ -5,6 +5,8 @@ import java.io.Closeable; import java.util.concurrent.CompletableFuture; /** + * Asynchronous feed client accepting document operations as JSON + * * @author bjorncs * @author jonmv */ diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index df1f3bcd54c..e0418836c80 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -45,8 +45,10 @@ public class FeedClientBuilder { PrivateKey privateKey; Collection caCertificates; + /** Creates a builder for a single container endpoint **/ public static FeedClientBuilder create(URI endpoint) { return new FeedClientBuilder(Collections.singletonList(endpoint)); } + /** Creates a builder for multiple container endpoints **/ public static FeedClientBuilder create(List endpoints) { return new FeedClientBuilder(endpoints); } private FeedClientBuilder(List endpoints) { @@ -86,61 +88,85 @@ public class FeedClientBuilder { return this; } + /** Sets {@link SSLContext} instance. */ public FeedClientBuilder setSslContext(SSLContext context) { this.sslContext = requireNonNull(context); return this; } + /** Sets {@link HostnameVerifier} instance (e.g for disabling default SSL hostname verification). */ public FeedClientBuilder setHostnameVerifier(HostnameVerifier verifier) { this.hostnameVerifier = requireNonNull(verifier); return this; } + /** Adds HTTP request header to all client requests. */ public FeedClientBuilder addRequestHeader(String name, String value) { return addRequestHeader(name, () -> requireNonNull(value)); } + /** + * Adds HTTP request header to all client requests. Value {@link Supplier} is invoked for each HTTP request, + * i.e. value can be dynamically updated during a feed. + */ public FeedClientBuilder addRequestHeader(String name, Supplier valueSupplier) { this.requestHeaders.put(requireNonNull(name), requireNonNull(valueSupplier)); return this; } + /** + * Overrides default retry strategy. + * @see FeedClient.RetryStrategy + */ public FeedClientBuilder setRetryStrategy(FeedClient.RetryStrategy strategy) { this.retryStrategy = requireNonNull(strategy); return this; } + /** + * Overrides default circuit breaker. + * @see FeedClient.CircuitBreaker + */ public FeedClientBuilder setCircuitBreaker(FeedClient.CircuitBreaker breaker) { this.circuitBreaker = requireNonNull(breaker); return this; } + /** Sets path to client SSL certificate/key PEM files */ public FeedClientBuilder setCertificate(Path certificatePemFile, Path privateKeyPemFile) { this.certificateFile = certificatePemFile; this.privateKeyFile = privateKeyPemFile; return this; } + /** Sets client SSL certificates/key */ public FeedClientBuilder setCertificate(Collection certificate, PrivateKey privateKey) { this.certificate = certificate; this.privateKey = privateKey; return this; } + /** Sets client SSL certificate/key */ public FeedClientBuilder setCertificate(X509Certificate certificate, PrivateKey privateKey) { return setCertificate(Collections.singletonList(certificate), privateKey); } + /** + * Overrides JVM default SSL truststore + * @param caCertificatesFile Path to PEM encoded file containing trusted certificates + */ public FeedClientBuilder setCaCertificatesFile(Path caCertificatesFile) { this.caCertificatesFile = caCertificatesFile; return this; } + /** Overrides JVM default SSL truststore */ public FeedClientBuilder setCaCertificates(Collection caCertificates) { this.caCertificates = caCertificates; return this; } + /** Constructs instance of {@link ai.vespa.feed.client.FeedClient} from builder configuration */ public FeedClient build() { try { validateConfiguration(); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedException.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedException.java index 25068818396..e1c6c733e9c 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedException.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedException.java @@ -2,6 +2,8 @@ package ai.vespa.feed.client; /** + * Signals that an error occurred during feeding + * * @author bjorncs */ public class FeedException extends RuntimeException { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonParseException.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonParseException.java index 785ffd8eb4c..8edf74ec275 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonParseException.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonParseException.java @@ -2,6 +2,8 @@ package ai.vespa.feed.client; /** + * Signals that supplied JSON is invalid + * * @author bjorncs */ public class JsonParseException extends FeedException { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/OperationParameters.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/OperationParameters.java index 22546f89ccb..8c20a37d224 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/OperationParameters.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/OperationParameters.java @@ -7,6 +7,8 @@ import java.util.Optional; import java.util.OptionalInt; /** + * Per-operation feed parameters + * * @author bjorncs * @author jonmv */ diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/Result.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/Result.java index 31a6cf6e893..b29d65e193b 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/Result.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/Result.java @@ -4,6 +4,8 @@ package ai.vespa.feed.client; import java.util.Optional; /** + * Result for a document operation + * * @author bjorncs * @author jonmv */ -- cgit v1.2.3 From 92b628bcf0976673756566054566ca3a3476cb66 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 14 Jun 2021 16:42:18 +0200 Subject: Package client with start script as zip artifact --- vespa-feed-client-cli/pom.xml | 37 +++++++++++++++------- .../src/main/sh/vespa-feed-client-standalone.sh | 9 ++++++ vespa-feed-client-cli/src/maven/create-zip.xml | 24 ++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) create mode 100755 vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh create mode 100644 vespa-feed-client-cli/src/maven/create-zip.xml diff --git a/vespa-feed-client-cli/pom.xml b/vespa-feed-client-cli/pom.xml index 9fd59f1cfa4..ebbea35f4a4 100644 --- a/vespa-feed-client-cli/pom.xml +++ b/vespa-feed-client-cli/pom.xml @@ -71,25 +71,38 @@ + org.apache.maven.plugins maven-assembly-plugin - - - - ai.vespa.feed.client.CliClient - - - - jar-with-dependencies - - false - - make-assembly + make-fatjar + prepare-package + + single + + + false + + + ai.vespa.feed.client.CliClient + + + + jar-with-dependencies + + + + + make-zip package single + + + src/maven/create-zip.xml + + diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh new file mode 100755 index 00000000000..57077205d18 --- /dev/null +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +exec java \ +-Djava.awt.headless=true \ +-Xms128m -Xmx2048m \ +--add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ +-Djava.util.logging.config.file=logging.properties \ +-cp vespa-feed-client-cli-jar-with-dependencies.jar ai.vespa.feed.client.CliClient "$@" diff --git a/vespa-feed-client-cli/src/maven/create-zip.xml b/vespa-feed-client-cli/src/maven/create-zip.xml new file mode 100644 index 00000000000..45bbbea9f2d --- /dev/null +++ b/vespa-feed-client-cli/src/maven/create-zip.xml @@ -0,0 +1,24 @@ + + + + zip + true + + + zip + + + + ${project.build.directory}/${project.artifactId}-jar-with-dependencies.jar + + + ${project.basedir}/src/main/sh/vespa-feed-client-standalone.sh + vespa-feed-client + + + ${project.basedir}/src/main/resources/logging.properties + + + -- cgit v1.2.3 From 67b7cdb8fefde4dc5843d646ce9fbd33acb37f7f Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 14 Jun 2021 16:51:31 +0000 Subject: allow multiple tensor types for same name * with several document types you can have fields with the same name but different tensor types that all should be allowed as target for nearestNeighbor operator. --- .../searchers/ValidateNearestNeighborSearcher.java | 44 ++++++++++++++-------- .../searchers/ValidateNearestNeighborTestCase.java | 26 ++++++++++++- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java index ca9d17cb656..d22dd2e6af6 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java @@ -19,6 +19,7 @@ import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.yolean.chain.Before; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,15 +33,17 @@ import java.util.Optional; @Before(GroupingExecutor.COMPONENT_NAME) // Must happen before query.prepare() public class ValidateNearestNeighborSearcher extends Searcher { - private final Map validAttributes = new HashMap<>(); + private final Map> validAttributes = new HashMap<>(); public ValidateNearestNeighborSearcher(AttributesConfig attributesConfig) { for (AttributesConfig.Attribute a : attributesConfig.attribute()) { - TensorType tt = null; + if (! validAttributes.containsKey(a.name())) { + validAttributes.put(a.name(), new ArrayList()); + } if (a.datatype() == AttributesConfig.Attribute.Datatype.TENSOR) { - tt = TensorType.fromSpec(a.tensortype()); + TensorType tt = TensorType.fromSpec(a.tensortype()); + validAttributes.get(a.name()).add(tt); } - validAttributes.put(a.name(), tt); } } @@ -60,10 +63,10 @@ public class ValidateNearestNeighborSearcher extends Searcher { public Optional errorMessage = Optional.empty(); - private final Map validAttributes; + private final Map> validAttributes; private final Query query; - public NNVisitor(RankProperties rankProperties, Map validAttributes, Query query) { + public NNVisitor(RankProperties rankProperties, Map> validAttributes, Query query) { this.validAttributes = validAttributes; this.query = query; } @@ -101,17 +104,26 @@ public class ValidateNearestNeighborSearcher extends Searcher { if (queryTensor.isEmpty()) return item + " requires a tensor rank feature " + queryFeatureName + " but this is not present"; - if ( ! validAttributes.containsKey(item.getIndexName())) + if ( ! validAttributes.containsKey(item.getIndexName())) { return item + " field is not an attribute"; - TensorType fieldType = validAttributes.get(item.getIndexName()); - if (fieldType == null) return item + " field is not a tensor"; - if ( ! isDenseVector(fieldType)) - return item + " tensor type " + fieldType + " is not a dense vector"; - - if ( ! isCompatible(fieldType, queryTensor.get().type())) - return item + " field type " + fieldType + " does not match query type " + queryTensor.get().type(); - - return null; + } + List allTensorTypes = validAttributes.get(item.getIndexName()); + for (TensorType fieldType : allTensorTypes) { + if (isDenseVector(fieldType) && isCompatible(fieldType, queryTensor.get().type())) { + return null; + } + } + for (TensorType fieldType : allTensorTypes) { + if (isDenseVector(fieldType) && ! isCompatible(fieldType, queryTensor.get().type())) { + return item + " field type " + fieldType + " does not match query type " + queryTensor.get().type(); + } + } + for (TensorType fieldType : allTensorTypes) { + if (! isDenseVector(fieldType)) { + return item + " tensor type " + fieldType + " is not a dense vector"; + } + } + return item + " field is not a tensor"; } @Override diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 72956b5b6eb..e5ed6f89fd4 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -51,10 +51,20 @@ public class ValidateNearestNeighborTestCase { "attribute[3].tensortype tensor(x{})\n" + "attribute[4].name matrix\n" + "attribute[4].datatype TENSOR\n" + - "attribute[4].tensortype tensor(x[3],y[1])\n" + "attribute[4].tensortype tensor(x[3],y[1])\n" + + "attribute[5].name threetypes\n" + + "attribute[5].datatype TENSOR\n" + + "attribute[5].tensortype tensor(x[42])\n" + + "attribute[6].name threetypes\n" + + "attribute[6].datatype TENSOR\n" + + "attribute[6].tensortype tensor(x[3])\n" + + "attribute[7].name threetypes\n" + + "attribute[7].datatype TENSOR\n" + + "attribute[7].tensortype tensor(x{})\n" ))); } + private static TensorType tt_dense_dvector_42 = TensorType.fromSpec("tensor(x[42])"); private static TensorType tt_dense_dvector_3 = TensorType.fromSpec("tensor(x[3])"); private static TensorType tt_dense_dvector_2 = TensorType.fromSpec("tensor(x[2])"); private static TensorType tt_dense_fvector_3 = TensorType.fromSpec("tensor(x[3])"); @@ -185,6 +195,20 @@ public class ValidateNearestNeighborTestCase { assertErrMsg(desc("simple", "qvector", 1, "field is not a tensor"), r); } + @Test + public void testSeveralAttributesWithSameName() { + String q = makeQuery("threetypes", "qvector"); + Tensor t1 = makeTensor(tt_dense_fvector_3); + Result r1 = doSearch(searcher, q, t1); + assertNull(r1.hits().getError()); + Tensor t2 = makeTensor(tt_dense_dvector_42, 42); + Result r2 = doSearch(searcher, q, t2); + assertNull(r2.hits().getError()); + Tensor t3 = makeTensor(tt_dense_dvector_2, 2); + Result r3 = doSearch(searcher, q, t3); + assertErrMsg(desc("threetypes", "qvector", 1, "field type tensor(x[42]) does not match query type tensor(x[2])"), r3); + } + @Test public void testSparseTensor() { String q = makeQuery("sparse", "qvector"); -- cgit v1.2.3 From c0ec68e1af6541694132ea0dd9a8fdf1fac3bcd1 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Tue, 15 Jun 2021 08:04:53 +0000 Subject: adapt overall timeout if user specifies ping timeout --- configd/src/apps/cmd/main.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configd/src/apps/cmd/main.cpp b/configd/src/apps/cmd/main.cpp index 7e5fbba1465..1b90483b65d 100644 --- a/configd/src/apps/cmd/main.cpp +++ b/configd/src/apps/cmd/main.cpp @@ -95,17 +95,17 @@ Cmd::run(const Method &cmd, const char *arg) FRT_RPCRequest *req = _server->supervisor().AllocRPCRequest(); req->SetMethodName(cmd.rpcMethod); + int pingTimeoutMs = 5000; if (cmd.needsTimeoutArg) { - int timeoutMs = 5000; if (arg) { - timeoutMs = atoi(arg); + pingTimeoutMs = atoi(arg); } - req->GetParams()->AddInt32(timeoutMs); + req->GetParams()->AddInt32(pingTimeoutMs); } else if (arg) { // one param req->GetParams()->AddString(arg); } - _target->InvokeSync(req, 10.0); + _target->InvokeSync(req, 2 * pingTimeoutMs * 0.001); if (req->IsError()) { fprintf(stderr, "vespa-sentinel-cmd '%s' error %d: %s\n", -- cgit v1.2.3 From 8690905f7bcb32b12ba70ed722ee1bd29d268811 Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Tue, 15 Jun 2021 08:04:22 +0000 Subject: drop empty buffers instead of using small buffers --- .../vespa/clustercontroller/core/rpc/RPCCommunicator.java | 2 +- .../com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java | 2 +- .../yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java | 2 +- .../main/java/com/yahoo/vespa/config/proxy/ProxyServer.java | 2 +- .../main/java/com/yahoo/vespa/config/JRTConnectionPool.java | 2 +- .../java/com/yahoo/container/jdisc/ConfiguredApplication.java | 2 +- .../yahoo/documentapi/messagebus/protocol/ExternPolicy.java | 2 +- jrt/src/com/yahoo/jrt/Supervisor.java | 10 ---------- jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java | 2 +- .../src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java | 2 +- .../ai/vespa/metricsproxy/service/ConfigSentinelClient.java | 2 +- .../yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java | 2 +- 12 files changed, 11 insertions(+), 21 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index bb45de37ce3..ec05ac1ed29 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -54,7 +54,7 @@ public class RPCCommunicator implements Communicator { private final int fleetControllerIndex; public static Supervisor createRealSupervisor() { - return new Supervisor(new Transport("rpc-communicator")).useSmallBuffers(); + return new Supervisor(new Transport("rpc-communicator")).setDropEmptyBuffers(true); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index 73597e995d4..ce710a29180 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -99,7 +99,7 @@ public class RpcServer { disconnect(); log.log(Level.FINE, () -> "Fleetcontroller " + fleetControllerIndex + ": Connecting RPC server."); if (supervisor != null) disconnect(); - supervisor = new Supervisor(new Transport("rpc" + port)).useSmallBuffers(); + supervisor = new Supervisor(new Transport("rpc" + port)).setDropEmptyBuffers(true); addMethods(); log.log(Level.FINE, () -> "Fleetcontroller " + fleetControllerIndex + ": Attempting to bind to port " + port); acceptor = supervisor.listen(new Spec(port)); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index b4e9a760d8e..3fa1b32cada 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -53,7 +53,7 @@ public class SlobrokClient implements NodeLookup { this.connectionSpecs = slobrokConnectionSpecs; shutdown(); supervisor = new Supervisor(new Transport("slobrok-client")); - supervisor.useSmallBuffers(); + supervisor.setDropEmptyBuffers(true); SlobrokList slist = new SlobrokList(); slist.setup(slobrokConnectionSpecs); mirror = new Mirror(supervisor, slist); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 047cec87ed7..5ad9fabcb61 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -56,7 +56,7 @@ public class ProxyServer implements Runnable { ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) { this.configSource = source; - supervisor = new Supervisor(new Transport("proxy-server", JRT_TRANSPORT_THREADS)).useSmallBuffers(); + supervisor = new Supervisor(new Transport("proxy-server", JRT_TRANSPORT_THREADS)).setDropEmptyBuffers(true); log.log(Level.FINE, () -> "Using config source '" + source); this.memoryCache = memoryCache; this.rpcServer = createRpcServer(spec); diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index 26eafb67c1b..18939e847f7 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -37,7 +37,7 @@ public class JRTConnectionPool implements ConnectionPool { private volatile JRTConnection currentConnection; public JRTConnectionPool(ConfigSourceSet sourceSet) { - supervisor = new Supervisor(new Transport("config-jrtpool-" + sourceSet.hashCode())).useSmallBuffers(); + supervisor = new Supervisor(new Transport("config-jrtpool-" + sourceSet.hashCode())).setDropEmptyBuffers(true); addSources(sourceSet); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index fe8668427f4..0e4c4446778 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -155,7 +155,7 @@ public final class ConfiguredApplication implements Application { if ( ! qrConfig.rpc().enabled()) return null; // 1. Set up RPC server - supervisor = new Supervisor(new Transport("slobrok")).useSmallBuffers(); + supervisor = new Supervisor(new Transport("slobrok")).setDropEmptyBuffers(true); Spec listenSpec = new Spec(qrConfig.rpc().port()); try { acceptor = supervisor.listen(listenSpec); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java index 94dfabb2c4f..26b7cb71f2d 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java @@ -59,7 +59,7 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy { pattern = args[1]; session = pattern.substring(pos); orb = new Supervisor(new Transport("externpolicy")); - orb.useSmallBuffers(); + orb.setDropEmptyBuffers(true); mirror = new Mirror(orb, slobroks); error = null; } diff --git a/jrt/src/com/yahoo/jrt/Supervisor.java b/jrt/src/com/yahoo/jrt/Supervisor.java index d7c2c83ea69..5975e191a5b 100644 --- a/jrt/src/com/yahoo/jrt/Supervisor.java +++ b/jrt/src/com/yahoo/jrt/Supervisor.java @@ -36,16 +36,6 @@ public class Supervisor { new MandatoryMethods(this); } - /** - * Will optimize buffers size for small memory footprint - * Use this when you have many connections with very little traffic. - **/ - public Supervisor useSmallBuffers() { - setMaxInputBufferSize(SMALL_INPUT_BUFFER_SIZE); - setMaxOutputBufferSize(SMALL_OUTPUT_BUFFER_SIZE); - return this; - } - /** * Drop empty buffers. This will reduce memory footprint for idle * connections at the cost of extra allocations when buffer space diff --git a/jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java b/jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java index 854cd973e4d..6d66a38406a 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java +++ b/jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java @@ -39,7 +39,7 @@ public class Slobrok { public Slobrok(int port) throws ListenFailedException { // NB: rpc must be single-threaded - orb = new Supervisor(new Transport("slobrok-" + port, 1)).useSmallBuffers(); + orb = new Supervisor(new Transport("slobrok-" + port, 1)).setDropEmptyBuffers(true); registerMethods(); try { listener = orb.listen(new Spec(port)); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java index 9d93b440a1d..881ed19ce0c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java @@ -27,7 +27,7 @@ public class RpcConnector extends AbstractComponent { private final Acceptor acceptor; public RpcConnector(RpcConnectorConfig config) { - supervisor = new Supervisor(new Transport("rpc-" + config.port())).useSmallBuffers(); + supervisor = new Supervisor(new Transport("rpc-" + config.port())).setDropEmptyBuffers(true); Spec spec = new Spec(config.port()); try { acceptor = supervisor.listen(spec); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index ab05e778ea6..d07a52f42bd 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -28,7 +28,7 @@ public class ConfigSentinelClient extends AbstractComponent { @Inject public ConfigSentinelClient() { - supervisor = new Supervisor(new Transport("sentinel-client")).useSmallBuffers(); + supervisor = new Supervisor(new Transport("sentinel-client")).setDropEmptyBuffers(true); } @Override diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 0c9148ad834..34e5fe49f69 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -50,7 +50,7 @@ public class SlobrokMonitorManagerImpl extends AbstractComponent implements Slob private SlobrokMonitorManagerImpl(Transport transport, Supervisor orb, DuperModelManager duperModel) { this(() -> new SlobrokMonitor(orb), transport, duperModel); - orb.useSmallBuffers(); + orb.setDropEmptyBuffers(true); } SlobrokMonitorManagerImpl(Supplier slobrokMonitorFactory, Transport transport, DuperModelManager duperModel) { -- cgit v1.2.3 From 815bfdc744d88b16f68de1460b8a8c0661ea8650 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 15 Jun 2021 10:44:32 +0200 Subject: Temporarily print error to stdout and stderr --- .../java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index 73e10c39419..15b3d2e9d7d 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -57,7 +57,10 @@ public class VespaRecordWriter extends RecordWriter { if (error instanceof JsonParseException) { counters.incrementDocumentsSkipped(1); } else { - log.warning("Failed to feed single document: " + error); + String msg = "Failed to feed single document: " + error; + System.out.println(msg); + System.err.println(msg); + log.warning(msg); counters.incrementDocumentsFailed(1); } } else { -- cgit v1.2.3 From 56bb744bc61288b5c9c35dbdf8b11f6aa1ac0baf Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 15 Jun 2021 10:50:16 +0200 Subject: Extend flag expiry --- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index e0f28357872..45c423f7353 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -158,7 +158,7 @@ public class Flags { public static final UnboundIntFlag METRICS_PROXY_MAX_HEAP_SIZE_IN_MB = defineIntFlag( "metrics-proxy-max-heap-size-in-mb", 256, - List.of("hmusum"), "2021-03-01", "2021-06-15", + List.of("hmusum"), "2021-03-01", "2021-07-01", "JVM max heap size for metrics proxy in Mb", "Takes effect when restarting metrics proxy", CLUSTER_TYPE); -- cgit v1.2.3 From 49ca12d8ff13e8801f4f0804b5f2aa11480266ed Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 15 Jun 2021 09:09:54 +0000 Subject: IndexSet -> IndexList and unify on 'using' directives. --- build_settings.cmake | 2 +- .../searchlib/attribute/enum_store_dictionary.cpp | 54 +++++++++++----------- .../searchlib/attribute/enum_store_dictionary.h | 38 ++++++++------- .../searchlib/attribute/i_enum_store_dictionary.h | 20 ++++---- 4 files changed, 55 insertions(+), 59 deletions(-) diff --git a/build_settings.cmake b/build_settings.cmake index 10f4c7ff926..068a69cc7e8 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -73,7 +73,7 @@ else() endif() # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} ${VESPA_XXHASH_DEFINE} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} ${VESPA_XXHASH_DEFINE} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") # AddressSanitizer/ThreadSanitizer work for both GCC and Clang if (VESPA_USE_SANITIZER) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${VESPA_USE_SANITIZER}") diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index 1b21f3a7b6e..3b3fdd9bc5c 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -10,7 +10,6 @@ #include LOG_SETUP(".searchlib.attribute.enum_store_dictionary"); -using vespalib::datastore::EntryComparator; using vespalib::datastore::EntryRef; using vespalib::datastore::UniqueStoreAddResult; @@ -20,8 +19,7 @@ using vespalib::btree::BTreeNode; template void -EnumStoreDictionary::remove_unused_values(const IndexSet& unused, - const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary::remove_unused_values(const IndexList & unused,const EntryComparator& cmp) { for (const auto& ref : unused) { this->remove(cmp, ref); @@ -40,9 +38,9 @@ EnumStoreDictionary::~EnumStoreDictionary() = template void -EnumStoreDictionary::free_unused_values(const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary::free_unused_values(const EntryComparator& cmp) { - IndexSet unused; + IndexList unused; // find unused enums if constexpr (has_btree_dictionary) { @@ -59,10 +57,9 @@ EnumStoreDictionary::free_unused_values(const template void -EnumStoreDictionary::free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary::free_unused_values(const IndexList& to_remove, const EntryComparator& cmp) { - IndexSet unused; + IndexList unused; EntryRef prev; for (const auto& index : to_remove) { @@ -96,7 +93,7 @@ EnumStoreDictionary::remove(const EntryCompar template bool -EnumStoreDictionary::find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const +EnumStoreDictionary::find_index(const EntryComparator& cmp, Index& idx) const { if constexpr (has_hash_dictionary) { auto find_result = this->_hash_dict.find(cmp, EntryRef()); @@ -117,7 +114,7 @@ EnumStoreDictionary::find_index(const vespali template bool -EnumStoreDictionary::find_frozen_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const +EnumStoreDictionary::find_frozen_index(const EntryComparator& cmp, Index& idx) const { if constexpr (has_hash_dictionary) { auto find_result = this->_hash_dict.find(cmp, EntryRef()); @@ -138,7 +135,7 @@ EnumStoreDictionary::find_frozen_index(const template std::vector -EnumStoreDictionary::find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const +EnumStoreDictionary::find_matching_enums(const EntryComparator& cmp) const { std::vector result; if constexpr (has_btree_dictionary) { @@ -169,14 +166,14 @@ EnumStoreDictionary::get_frozen_root() const template <> std::pair -EnumStoreDictionary::find_posting_list(const vespalib::datastore::EntryComparator&, EntryRef) const +EnumStoreDictionary::find_posting_list(const EntryComparator&, EntryRef) const { LOG_ABORT("should not be reached"); } template std::pair -EnumStoreDictionary::find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const +EnumStoreDictionary::find_posting_list(const EntryComparator& cmp, EntryRef root) const { if constexpr (has_hash_dictionary) { (void) root; @@ -197,7 +194,7 @@ EnumStoreDictionary::find_posting_list(const template void -EnumStoreDictionary::collect_folded(Index idx, EntryRef, const std::function& callback) const +EnumStoreDictionary::collect_folded(Index idx, EntryRef, const std::function& callback) const { callback(idx); } @@ -242,14 +239,14 @@ EnumStoreDictionary::clear_all_posting_lists( template <> void -EnumStoreDictionary::update_posting_list(Index, const vespalib::datastore::EntryComparator&, std::function) +EnumStoreDictionary::update_posting_list(Index, const EntryComparator&, std::function) { LOG_ABORT("should not be reached"); } template void -EnumStoreDictionary::update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function updater) +EnumStoreDictionary::update_posting_list(Index idx, const EntryComparator& cmp, std::function updater) { if constexpr (has_btree_dictionary) { auto& dict = this->_btree_dict; @@ -334,7 +331,7 @@ EnumStoreDictionary::get_posting_dictionary() return this->_btree_dict; } -EnumStoreFoldedDictionary::EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr compare, std::unique_ptr folded_compare) +EnumStoreFoldedDictionary::EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr compare, std::unique_ptr folded_compare) : EnumStoreDictionary(enumStore, std::move(compare)), _folded_compare(std::move(folded_compare)) { @@ -387,7 +384,7 @@ EnumStoreFoldedDictionary::remove(const EntryComparator& comp, EntryRef ref) } void -EnumStoreFoldedDictionary::collect_folded(Index idx, EntryRef root, const std::function& callback) const +EnumStoreFoldedDictionary::collect_folded(Index idx, EntryRef root, const std::function& callback) const { BTreeDictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), _btree_dict.getAllocator()); itr.lower_bound(root, idx, *_folded_compare); @@ -419,6 +416,7 @@ namespace vespalib::btree { using search::IEnumStore; using search::EnumTreeTraits; +using datastore::EntryComparatorWrapper; template class BTreeNodeT; @@ -454,19 +452,19 @@ class BTreeNodeStore; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRoot; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootT; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootT; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootBase; template class BTreeConstIterator; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeConstIterator; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeIterator; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeIterator; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTree; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTree; + const EntryComparatorWrapper, EnumTreeTraits>; } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index a39ff524618..3626fb098d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -18,9 +18,10 @@ protected: using EntryRef = IEnumStoreDictionary::EntryRef; using Index = IEnumStoreDictionary::Index; using BTreeDictionaryType = BTreeDictionaryT; + using EntryComparator = IEnumStoreDictionary::EntryComparator; private: using EnumVector = IEnumStoreDictionary::EnumVector; - using IndexSet = IEnumStoreDictionary::IndexSet; + using IndexList = IEnumStoreDictionary::IndexList; using IndexVector = IEnumStoreDictionary::IndexVector; using ParentUniqueStoreDictionary = vespalib::datastore::UniqueStoreDictionary; using generation_t = IEnumStoreDictionary::generation_t; @@ -30,31 +31,28 @@ protected: private: IEnumStore& _enumStore; - void remove_unused_values(const IndexSet& unused, - const vespalib::datastore::EntryComparator& cmp); + void remove_unused_values(const IndexList& unused, const EntryComparator& cmp); public: - EnumStoreDictionary(IEnumStore& enumStore, std::unique_ptr compare); + EnumStoreDictionary(IEnumStore& enumStore, std::unique_ptr compare); ~EnumStoreDictionary() override; - void free_unused_values(const vespalib::datastore::EntryComparator& cmp) override; + void free_unused_values(const EntryComparator& cmp) override; + void free_unused_values(const IndexList& to_remove, const EntryComparator& cmp) override; - void free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) override; - - void remove(const vespalib::datastore::EntryComparator& comp, vespalib::datastore::EntryRef ref) override; - bool find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const override; - bool find_frozen_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const override; + void remove(const EntryComparator& comp, EntryRef ref) override; + bool find_index(const EntryComparator& cmp, Index& idx) const override; + bool find_frozen_index(const EntryComparator& cmp, Index& idx) const override; std::vector - find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const override; + find_matching_enums(const EntryComparator& cmp) const override; EntryRef get_frozen_root() const override; - std::pair find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const override; - void collect_folded(Index idx, EntryRef root, const std::function& callback) const override; + std::pair find_posting_list(const EntryComparator& cmp, EntryRef root) const override; + void collect_folded(Index idx, EntryRef root, const std::function& callback) const override; Index remap_index(Index idx) override; void clear_all_posting_lists(std::function clearer) override; - void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function updater) override; + void update_posting_list(Index idx, const EntryComparator& cmp, std::function updater) override; bool normalize_posting_lists(std::function normalize) override; const EnumPostingTree& get_posting_dictionary() const override; }; @@ -71,14 +69,14 @@ public: class EnumStoreFoldedDictionary : public EnumStoreDictionary { private: - std::unique_ptr _folded_compare; + std::unique_ptr _folded_compare; public: - EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr compare, std::unique_ptr folded_compare); + EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr compare, std::unique_ptr folded_compare); ~EnumStoreFoldedDictionary() override; - vespalib::datastore::UniqueStoreAddResult add(const vespalib::datastore::EntryComparator& comp, std::function insertEntry) override; - void remove(const vespalib::datastore::EntryComparator& comp, vespalib::datastore::EntryRef ref) override; - void collect_folded(Index idx, EntryRef root, const std::function& callback) const override; + vespalib::datastore::UniqueStoreAddResult add(const EntryComparator& comp, std::function insertEntry) override; + void remove(const EntryComparator& comp, EntryRef ref) override; + void collect_folded(Index idx, EntryRef root, const std::function& callback) const override; Index remap_index(Index idx) override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index 9d72369f245..bef7384b0b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -29,29 +29,29 @@ using EnumPostingTree = vespalib::btree::BTree - find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const = 0; + find_matching_enums(const EntryComparator& cmp) const = 0; virtual EntryRef get_frozen_root() const = 0; - virtual std::pair find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const = 0; - virtual void collect_folded(Index idx, EntryRef root, const std::function& callback) const = 0; + virtual std::pair find_posting_list(const EntryComparator& cmp, EntryRef root) const = 0; + virtual void collect_folded(Index idx, EntryRef root, const std::function& callback) const = 0; virtual Index remap_index(Index idx) = 0; virtual void clear_all_posting_lists(std::function clearer) = 0; - virtual void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function updater) = 0; + virtual void update_posting_list(Index idx, const EntryComparator& cmp, std::function updater) = 0; virtual bool normalize_posting_lists(std::function normalize) = 0; virtual const EnumPostingTree& get_posting_dictionary() const = 0; }; -- cgit v1.2.3 From 92304a9f760e6247717cc467c7ec953f48f4cb90 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 15 Jun 2021 09:35:22 +0000 Subject: Revert unintentional change. --- build_settings.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_settings.cmake b/build_settings.cmake index 068a69cc7e8..10f4c7ff926 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -73,7 +73,7 @@ else() endif() # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} ${VESPA_XXHASH_DEFINE} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} ${VESPA_XXHASH_DEFINE} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") # AddressSanitizer/ThreadSanitizer work for both GCC and Clang if (VESPA_USE_SANITIZER) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${VESPA_USE_SANITIZER}") -- cgit v1.2.3