diff options
47 files changed, 468 insertions, 232 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index bb45de37ce3..ec05ac1ed29 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -54,7 +54,7 @@ public class RPCCommunicator implements Communicator { private final int fleetControllerIndex; public static Supervisor createRealSupervisor() { - return new Supervisor(new Transport("rpc-communicator")).useSmallBuffers(); + return new Supervisor(new Transport("rpc-communicator")).setDropEmptyBuffers(true); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index 73597e995d4..ce710a29180 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -99,7 +99,7 @@ public class RpcServer { disconnect(); log.log(Level.FINE, () -> "Fleetcontroller " + fleetControllerIndex + ": Connecting RPC server."); if (supervisor != null) disconnect(); - supervisor = new Supervisor(new Transport("rpc" + port)).useSmallBuffers(); + supervisor = new Supervisor(new Transport("rpc" + port)).setDropEmptyBuffers(true); addMethods(); log.log(Level.FINE, () -> "Fleetcontroller " + fleetControllerIndex + ": Attempting to bind to port " + port); acceptor = supervisor.listen(new Spec(port)); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index b4e9a760d8e..3fa1b32cada 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -53,7 +53,7 @@ public class SlobrokClient implements NodeLookup { this.connectionSpecs = slobrokConnectionSpecs; shutdown(); supervisor = new Supervisor(new Transport("slobrok-client")); - supervisor.useSmallBuffers(); + supervisor.setDropEmptyBuffers(true); SlobrokList slist = new SlobrokList(); slist.setup(slobrokConnectionSpecs); mirror = new Mirror(supervisor, slist); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 047cec87ed7..5ad9fabcb61 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -56,7 +56,7 @@ public class ProxyServer implements Runnable { ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) { this.configSource = source; - supervisor = new Supervisor(new Transport("proxy-server", JRT_TRANSPORT_THREADS)).useSmallBuffers(); + supervisor = new Supervisor(new Transport("proxy-server", JRT_TRANSPORT_THREADS)).setDropEmptyBuffers(true); log.log(Level.FINE, () -> "Using config source '" + source); this.memoryCache = memoryCache; this.rpcServer = createRpcServer(spec); diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index e5bcb56ccb3..ef6ff3a0327 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -43,7 +43,7 @@ public class JRTConnectionPool implements ConnectionPool { public JRTConnectionPool(ConfigSourceSet sourceSet, String poolName) { this.poolName = poolName; - supervisor = new Supervisor(new Transport(poolName)).useSmallBuffers(); + supervisor = new Supervisor(new Transport(poolName))..setDropEmptyBuffers(true); addSources(sourceSet); } diff --git a/configd/src/apps/cmd/main.cpp b/configd/src/apps/cmd/main.cpp index 33b4aa8111d..1b90483b65d 100644 --- a/configd/src/apps/cmd/main.cpp +++ b/configd/src/apps/cmd/main.cpp @@ -13,6 +13,24 @@ #include <vespa/log/log.h> LOG_SETUP("vespa-sentinel-cmd"); +namespace { +struct Method { + const char * name; + const char * rpcMethod; + bool noArgNeeded; + bool needsTimeoutArg; +}; +const Method methods[] = { + { "list", "sentinel.ls", true, false }, + { "restart", "sentinel.service.restart", false, false }, + { "start", "sentinel.service.start", false, false }, + { "stop", "sentinel.service.stop", false, false }, + { "connectivity", "sentinel.report.connectivity", true, true } +}; + +} + + class Cmd { private: @@ -22,7 +40,7 @@ private: public: Cmd() : _server(), _target(nullptr) {} ~Cmd(); - int run(const char *cmd, const char *arg); + int run(const Method &cmd, const char *arg); void initRPC(const char *spec); void finiRPC(); }; @@ -41,6 +59,7 @@ void usage() fprintf(stderr, " restart {service}\n"); fprintf(stderr, " start {service}\n"); fprintf(stderr, " stop {service}\n"); + fprintf(stderr, " connectivity [milliseconds]\n"); } void @@ -63,7 +82,7 @@ Cmd::finiRPC() int -Cmd::run(const char *cmd, const char *arg) +Cmd::run(const Method &cmd, const char *arg) { int retval = 0; try { @@ -74,33 +93,61 @@ Cmd::run(const char *cmd, const char *arg) return 2; } FRT_RPCRequest *req = _server->supervisor().AllocRPCRequest(); - req->SetMethodName(cmd); + req->SetMethodName(cmd.rpcMethod); - if (arg) { + int pingTimeoutMs = 5000; + if (cmd.needsTimeoutArg) { + if (arg) { + pingTimeoutMs = atoi(arg); + } + req->GetParams()->AddInt32(pingTimeoutMs); + } else if (arg) { // one param req->GetParams()->AddString(arg); } - _target->InvokeSync(req, 5.0); + _target->InvokeSync(req, 2 * pingTimeoutMs * 0.001); if (req->IsError()) { fprintf(stderr, "vespa-sentinel-cmd '%s' error %d: %s\n", - cmd, req->GetErrorCode(), req->GetErrorMessage()); + cmd.name, req->GetErrorCode(), req->GetErrorMessage()); retval = 1; } else { FRT_Values &answer = *(req->GetReturn()); const char *atypes = answer.GetTypeString(); - fprintf(stderr, "vespa-sentinel-cmd '%s' OK.\n", cmd); - uint32_t idx = 0; - while (atypes != nullptr && *atypes != '\0') { - switch (*atypes) { - case 's': + fprintf(stderr, "vespa-sentinel-cmd '%s' OK.\n", cmd.name); + if (atypes && (strcmp(atypes, "SS") == 0)) { + uint32_t numHosts = answer[0]._string_array._len; + uint32_t numStats = answer[1]._string_array._len; + FRT_StringValue *hosts = answer[0]._string_array._pt; + FRT_StringValue *stats = answer[1]._string_array._pt; + uint32_t ml = 0; + uint32_t j; + for (j = 0; j < numHosts; ++j) { + uint32_t hl = strlen(hosts[j]._str); + if (hl > ml) ml = hl; + } + for (j = 0; j < numHosts && j < numStats; ++j) { + printf("%-*s -> %s\n", ml, hosts[j]._str, stats[j]._str); + } + for (; j < numHosts; ++j) { + printf("Extra host: %s\n", hosts[j]._str); + } + for (; j < numStats; ++j) { + printf("Extra stat: %s\n", stats[j]._str); + } + } else { + uint32_t idx = 0; + while (atypes != nullptr && *atypes != '\0') { + switch (*atypes) { + case 's': printf("%s\n", answer[idx]._string._str); break; - default: + default: printf("BAD: unknown type %c\n", *atypes); - } - ++atypes; + } + ++atypes; ++idx; + } } } req->SubRef(); @@ -108,19 +155,15 @@ Cmd::run(const char *cmd, const char *arg) return retval; } -const char * +const Method * parseCmd(const char *arg) { - if (strcmp(arg, "list") == 0) { - return "sentinel.ls"; - } else if (strcmp(arg, "restart") == 0) { - return "sentinel.service.restart"; - } else if (strcmp(arg, "start") == 0) { - return "sentinel.service.start"; - } else if (strcmp(arg, "stop") == 0) { - return "sentinel.service.stop"; + for (const auto & method : methods) { + if (strcmp(arg, method.name) == 0) { + return &method; + } } - return 0; + return nullptr; } void hookSignals() { @@ -131,14 +174,15 @@ void hookSignals() { int main(int argc, char** argv) { int retval = 1; - const char *cmd = 0; + const Method *cmd = nullptr; if (argc > 1) { cmd = parseCmd(argv[1]); } - if (cmd) { + const char *extraArg = (argc > 2 ? argv[2] : nullptr); + if (cmd && (extraArg || cmd->noArgNeeded)) { hookSignals(); Cmd runner; - retval = runner.run(cmd, argc > 2 ? argv[2] : 0); + retval = runner.run(*cmd, extraArg); } else { usage(); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index fe8668427f4..0e4c4446778 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -155,7 +155,7 @@ public final class ConfiguredApplication implements Application { if ( ! qrConfig.rpc().enabled()) return null; // 1. Set up RPC server - supervisor = new Supervisor(new Transport("slobrok")).useSmallBuffers(); + supervisor = new Supervisor(new Transport("slobrok")).setDropEmptyBuffers(true); Spec listenSpec = new Spec(qrConfig.rpc().port()); try { acceptor = supervisor.listen(listenSpec); diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java index ca9d17cb656..d22dd2e6af6 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java @@ -19,6 +19,7 @@ import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.yolean.chain.Before; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,15 +33,17 @@ import java.util.Optional; @Before(GroupingExecutor.COMPONENT_NAME) // Must happen before query.prepare() public class ValidateNearestNeighborSearcher extends Searcher { - private final Map<String, TensorType> validAttributes = new HashMap<>(); + private final Map<String, List<TensorType>> validAttributes = new HashMap<>(); public ValidateNearestNeighborSearcher(AttributesConfig attributesConfig) { for (AttributesConfig.Attribute a : attributesConfig.attribute()) { - TensorType tt = null; + if (! validAttributes.containsKey(a.name())) { + validAttributes.put(a.name(), new ArrayList<TensorType>()); + } if (a.datatype() == AttributesConfig.Attribute.Datatype.TENSOR) { - tt = TensorType.fromSpec(a.tensortype()); + TensorType tt = TensorType.fromSpec(a.tensortype()); + validAttributes.get(a.name()).add(tt); } - validAttributes.put(a.name(), tt); } } @@ -60,10 +63,10 @@ public class ValidateNearestNeighborSearcher extends Searcher { public Optional<ErrorMessage> errorMessage = Optional.empty(); - private final Map<String, TensorType> validAttributes; + private final Map<String, List<TensorType>> validAttributes; private final Query query; - public NNVisitor(RankProperties rankProperties, Map<String, TensorType> validAttributes, Query query) { + public NNVisitor(RankProperties rankProperties, Map<String, List<TensorType>> validAttributes, Query query) { this.validAttributes = validAttributes; this.query = query; } @@ -101,17 +104,26 @@ public class ValidateNearestNeighborSearcher extends Searcher { if (queryTensor.isEmpty()) return item + " requires a tensor rank feature " + queryFeatureName + " but this is not present"; - if ( ! validAttributes.containsKey(item.getIndexName())) + if ( ! validAttributes.containsKey(item.getIndexName())) { return item + " field is not an attribute"; - TensorType fieldType = validAttributes.get(item.getIndexName()); - if (fieldType == null) return item + " field is not a tensor"; - if ( ! isDenseVector(fieldType)) - return item + " tensor type " + fieldType + " is not a dense vector"; - - if ( ! isCompatible(fieldType, queryTensor.get().type())) - return item + " field type " + fieldType + " does not match query type " + queryTensor.get().type(); - - return null; + } + List<TensorType> allTensorTypes = validAttributes.get(item.getIndexName()); + for (TensorType fieldType : allTensorTypes) { + if (isDenseVector(fieldType) && isCompatible(fieldType, queryTensor.get().type())) { + return null; + } + } + for (TensorType fieldType : allTensorTypes) { + if (isDenseVector(fieldType) && ! isCompatible(fieldType, queryTensor.get().type())) { + return item + " field type " + fieldType + " does not match query type " + queryTensor.get().type(); + } + } + for (TensorType fieldType : allTensorTypes) { + if (! isDenseVector(fieldType)) { + return item + " tensor type " + fieldType + " is not a dense vector"; + } + } + return item + " field is not a tensor"; } @Override diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 72956b5b6eb..e5ed6f89fd4 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -51,10 +51,20 @@ public class ValidateNearestNeighborTestCase { "attribute[3].tensortype tensor(x{})\n" + "attribute[4].name matrix\n" + "attribute[4].datatype TENSOR\n" + - "attribute[4].tensortype tensor(x[3],y[1])\n" + "attribute[4].tensortype tensor(x[3],y[1])\n" + + "attribute[5].name threetypes\n" + + "attribute[5].datatype TENSOR\n" + + "attribute[5].tensortype tensor(x[42])\n" + + "attribute[6].name threetypes\n" + + "attribute[6].datatype TENSOR\n" + + "attribute[6].tensortype tensor(x[3])\n" + + "attribute[7].name threetypes\n" + + "attribute[7].datatype TENSOR\n" + + "attribute[7].tensortype tensor(x{})\n" ))); } + private static TensorType tt_dense_dvector_42 = TensorType.fromSpec("tensor(x[42])"); private static TensorType tt_dense_dvector_3 = TensorType.fromSpec("tensor(x[3])"); private static TensorType tt_dense_dvector_2 = TensorType.fromSpec("tensor(x[2])"); private static TensorType tt_dense_fvector_3 = TensorType.fromSpec("tensor<float>(x[3])"); @@ -186,6 +196,20 @@ public class ValidateNearestNeighborTestCase { } @Test + public void testSeveralAttributesWithSameName() { + String q = makeQuery("threetypes", "qvector"); + Tensor t1 = makeTensor(tt_dense_fvector_3); + Result r1 = doSearch(searcher, q, t1); + assertNull(r1.hits().getError()); + Tensor t2 = makeTensor(tt_dense_dvector_42, 42); + Result r2 = doSearch(searcher, q, t2); + assertNull(r2.hits().getError()); + Tensor t3 = makeTensor(tt_dense_dvector_2, 2); + Result r3 = doSearch(searcher, q, t3); + assertErrMsg(desc("threetypes", "qvector", 1, "field type tensor(x[42]) does not match query type tensor(x[2])"), r3); + } + + @Test public void testSparseTensor() { String q = makeQuery("sparse", "qvector"); Tensor t = makeTensor(tt_sparse_vector_x); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java index 94dfabb2c4f..26b7cb71f2d 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java @@ -59,7 +59,7 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy { pattern = args[1]; session = pattern.substring(pos); orb = new Supervisor(new Transport("externpolicy")); - orb.useSmallBuffers(); + orb.setDropEmptyBuffers(true); mirror = new Mirror(orb, slobroks); error = null; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index e0f28357872..45c423f7353 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -158,7 +158,7 @@ public class Flags { public static final UnboundIntFlag METRICS_PROXY_MAX_HEAP_SIZE_IN_MB = defineIntFlag( "metrics-proxy-max-heap-size-in-mb", 256, - List.of("hmusum"), "2021-03-01", "2021-06-15", + List.of("hmusum"), "2021-03-01", "2021-07-01", "JVM max heap size for metrics proxy in Mb", "Takes effect when restarting metrics proxy", CLUSTER_TYPE); diff --git a/jrt/src/com/yahoo/jrt/Supervisor.java b/jrt/src/com/yahoo/jrt/Supervisor.java index d7c2c83ea69..5975e191a5b 100644 --- a/jrt/src/com/yahoo/jrt/Supervisor.java +++ b/jrt/src/com/yahoo/jrt/Supervisor.java @@ -37,16 +37,6 @@ public class Supervisor { } /** - * Will optimize buffers size for small memory footprint - * Use this when you have many connections with very little traffic. - **/ - public Supervisor useSmallBuffers() { - setMaxInputBufferSize(SMALL_INPUT_BUFFER_SIZE); - setMaxOutputBufferSize(SMALL_OUTPUT_BUFFER_SIZE); - return this; - } - - /** * Drop empty buffers. This will reduce memory footprint for idle * connections at the cost of extra allocations when buffer space * is needed again. diff --git a/jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java b/jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java index 854cd973e4d..6d66a38406a 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java +++ b/jrt/src/com/yahoo/jrt/slobrok/server/Slobrok.java @@ -39,7 +39,7 @@ public class Slobrok { public Slobrok(int port) throws ListenFailedException { // NB: rpc must be single-threaded - orb = new Supervisor(new Transport("slobrok-" + port, 1)).useSmallBuffers(); + orb = new Supervisor(new Transport("slobrok-" + port, 1)).setDropEmptyBuffers(true); registerMethods(); try { listener = orb.listen(new Spec(port)); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java index 9d93b440a1d..881ed19ce0c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java @@ -27,7 +27,7 @@ public class RpcConnector extends AbstractComponent { private final Acceptor acceptor; public RpcConnector(RpcConnectorConfig config) { - supervisor = new Supervisor(new Transport("rpc-" + config.port())).useSmallBuffers(); + supervisor = new Supervisor(new Transport("rpc-" + config.port())).setDropEmptyBuffers(true); Spec spec = new Spec(config.port()); try { acceptor = supervisor.listen(spec); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index ab05e778ea6..d07a52f42bd 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -28,7 +28,7 @@ public class ConfigSentinelClient extends AbstractComponent { @Inject public ConfigSentinelClient() { - supervisor = new Supervisor(new Transport("sentinel-client")).useSmallBuffers(); + supervisor = new Supervisor(new Transport("sentinel-client")).setDropEmptyBuffers(true); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index fba0993f2f9..40f9b330634 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -29,10 +29,7 @@ public interface LoadBalancerService { Protocol protocol(); /** Returns whether load balancers created by this service can forward traffic to given node and cluster type */ - default boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { - return (nodeType == NodeType.tenant && clusterType.isContainer()) || - nodeType.isConfigServerLike(); - } + boolean supports(NodeType nodeType, ClusterSpec.Type clusterType); /** Load balancer protocols */ enum Protocol { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index b912087da46..f752cbc4349 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; import java.util.Collections; import java.util.HashMap; @@ -18,6 +19,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { private final Map<LoadBalancerId, LoadBalancerInstance> instances = new HashMap<>(); private boolean throwOnCreate = false; + private boolean supportsProvisioning = true; public Map<LoadBalancerId, LoadBalancerInstance> instances() { return Collections.unmodifiableMap(instances); @@ -28,6 +30,18 @@ public class LoadBalancerServiceMock implements LoadBalancerService { return this; } + public LoadBalancerServiceMock supportsProvisioning(boolean supportsProvisioning) { + this.supportsProvisioning = supportsProvisioning; + return this; + } + + @Override + public boolean supports(NodeType nodeType, ClusterSpec.Type clusterType) { + if (!supportsProvisioning) return false; + return (nodeType == NodeType.tenant && clusterType.isContainer()) || + nodeType.isConfigServerLike(); + } + @Override public Protocol protocol() { return Protocol.ipv4; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java index 7667672e470..9a6a65eca69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeType; import java.util.Comparator; import java.util.Optional; @@ -35,4 +36,9 @@ public class PassthroughLoadBalancerService implements LoadBalancerService { return Protocol.ipv4; } + @Override + public boolean supports(NodeType nodeType, ClusterSpec.Type clusterType) { + return true; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 33a3c138d70..e17e5a5a449 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -66,7 +66,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { + public boolean supports(NodeType nodeType, ClusterSpec.Type clusterType) { // Shared routing layer only supports routing to tenant nodes return nodeType == NodeType.tenant && clusterType.isContainer(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java index 80f74a011c0..6d88e43630a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostEncrypter.java @@ -48,6 +48,8 @@ public class HostEncrypter extends NodeRepositoryMaintainer { NodeList allNodes = nodeRepository().nodes().list(); for (var nodeType : NodeType.values()) { if (!nodeType.isHost()) continue; + // TODO: Require a minimum number of proxies in Orchestrator. For now skip proxy hosts. + if (nodeType == NodeType.proxyhost) continue; if (upgradingVespa(allNodes, nodeType)) continue; unencryptedHosts(allNodes, nodeType).forEach(host -> encrypt(host, now)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index ac9d8d6671a..c3d6f5c42b8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -65,7 +65,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { Instant now = nodeRepository().clock().instant(); Instant expiry = now.minus(reservedExpiry); patchLoadBalancers(lb -> lb.state() == State.reserved && lb.changedAt().isBefore(expiry), - lb -> db.writeLoadBalancer(lb.with(State.inactive, now))); + lb -> db.writeLoadBalancer(lb.with(State.inactive, now), lb.state())); } /** Deprovision inactive load balancers that have expired */ @@ -114,7 +114,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { attempts.add(1); LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals)); service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals), true); - db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals)))); + db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals))), lb.state()); } catch (Exception e) { failed.add(lb.id()); lastException.set(e); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 364af6308a1..0205cc6c818 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -482,18 +482,29 @@ public class CuratorDatabaseClient { return read(loadBalancerPath(id), LoadBalancerSerializer::fromJson); } - public void writeLoadBalancer(LoadBalancer loadBalancer) { + public void writeLoadBalancer(LoadBalancer loadBalancer, LoadBalancer.State fromState) { NestedTransaction transaction = new NestedTransaction(); - writeLoadBalancers(List.of(loadBalancer), transaction); + writeLoadBalancers(List.of(loadBalancer), fromState, transaction); transaction.commit(); } - public void writeLoadBalancers(Collection<LoadBalancer> loadBalancers, NestedTransaction transaction) { + public void writeLoadBalancers(Collection<LoadBalancer> loadBalancers, LoadBalancer.State fromState, NestedTransaction transaction) { CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); loadBalancers.forEach(loadBalancer -> { curatorTransaction.add(createOrSet(loadBalancerPath(loadBalancer.id()), LoadBalancerSerializer.toJson(loadBalancer))); }); + transaction.onCommitted(() -> { + for (var lb : loadBalancers) { + if (lb.state() == fromState) continue; + if (fromState == null) { + log.log(Level.INFO, () -> "Creating " + lb.id() + " in " + lb.state()); + } else { + log.log(Level.INFO, () -> "Moving " + lb.id() + " from " + fromState + + " to " + lb.state()); + } + } + }); } public void removeLoadBalancer(LoadBalancerId loadBalancer) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index c114aa58a05..b6600574f94 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.NestedTransaction; @@ -61,7 +60,7 @@ public class LoadBalancerProvisioner { for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { var loadBalancer = db.readLoadBalancer(id); - loadBalancer.ifPresent(db::writeLoadBalancer); + loadBalancer.ifPresent(lb -> db.writeLoadBalancer(lb, lb.state())); } } } @@ -77,15 +76,13 @@ public class LoadBalancerProvisioner { * Calling this for irrelevant node or cluster types is a no-op. */ public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { - if (!service.canForwardTo(requestedNodes.type(), cluster.type())) return; // Nothing to provision for this node and cluster type + if (!service.supports(requestedNodes.type(), cluster.type())) return; // Nothing to provision for this node and cluster type if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lock(application)) { ClusterSpec.Id clusterId = effectiveId(cluster); - NodeList nodes = nodesOf(clusterId, application); LoadBalancerId loadBalancerId = requireNonClashing(new LoadBalancerId(application, clusterId)); - ApplicationTransaction transaction = new ApplicationTransaction(new ProvisionLock(application, lock), new NestedTransaction()); - provision(transaction, loadBalancerId, nodes, false); - transaction.nested().commit(); + NodeList nodes = nodesOf(clusterId, application); + prepare(loadBalancerId, nodes); } } @@ -100,14 +97,18 @@ public class LoadBalancerProvisioner { * Calling this when no load balancer has been prepared for given cluster is a no-op. */ public void activate(Set<ClusterSpec> clusters, ApplicationTransaction transaction) { + Set<ClusterSpec.Id> activatingClusters = clusters.stream() + .map(LoadBalancerProvisioner::effectiveId) + .collect(Collectors.toSet()); for (var cluster : loadBalancedClustersOf(transaction.application()).entrySet()) { - // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(transaction, cluster.getKey(), cluster.getValue()); + if (!activatingClusters.contains(cluster.getKey())) continue; + + Node clusterNode = cluster.getValue().first().get(); + if (!service.supports(clusterNode.type(), clusterNode.allocation().get().membership().cluster().type())) continue; + activate(transaction, cluster.getKey(), cluster.getValue()); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), clusters.stream() - .map(LoadBalancerProvisioner::effectiveId) - .collect(Collectors.toSet())); + var surplusLoadBalancers = surplusLoadBalancersOf(transaction.application(), activatingClusters); deactivate(surplusLoadBalancers, transaction.nested()); } @@ -140,7 +141,7 @@ public class LoadBalancerProvisioner { var deactivatedLoadBalancers = loadBalancers.stream() .map(lb -> lb.with(LoadBalancer.State.inactive, now)) .collect(Collectors.toList()); - db.writeLoadBalancers(deactivatedLoadBalancers, transaction); + db.writeLoadBalancers(deactivatedLoadBalancers, LoadBalancer.State.active, transaction); } /** Find all load balancer IDs owned by given tenant and application */ @@ -165,52 +166,41 @@ public class LoadBalancerProvisioner { return loadBalancerId; } - /** Idempotently provision a load balancer for given application and cluster */ - private void provision(ApplicationTransaction transaction, LoadBalancerId id, NodeList nodes, boolean activate) { + private void prepare(LoadBalancerId id, NodeList nodes) { Instant now = nodeRepository.clock().instant(); Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty() && activate) return; // Nothing to activate as this load balancer was never prepared - - Set<Real> reals = realsOf(nodes); - Optional<LoadBalancerInstance> instance = provisionInstance(id, reals, loadBalancer); + Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer); LoadBalancer newLoadBalancer; + LoadBalancer.State fromState = null; if (loadBalancer.isEmpty()) { newLoadBalancer = new LoadBalancer(id, instance, LoadBalancer.State.reserved, now); } else { - LoadBalancer.State state = activate && instance.isPresent() - ? LoadBalancer.State.active - : loadBalancer.get().state(); - newLoadBalancer = loadBalancer.get().with(instance).with(state, now); - if (loadBalancer.get().state() != newLoadBalancer.state()) { - log.log(Level.INFO, () -> "Moving " + newLoadBalancer.id() + " from " + loadBalancer.get().state() + - " to " + newLoadBalancer.state()); - } - } - - if (activate) { - db.writeLoadBalancers(List.of(newLoadBalancer), transaction.nested()); - } else { - // Always store load balancer so that LoadBalancerExpirer can expire partially provisioned load balancers - db.writeLoadBalancer(newLoadBalancer); - } - - // Signal that load balancer is not ready yet - if (instance.isEmpty()) { - throw new LoadBalancerServiceException("Could not (re)configure " + id + ", targeting: " + - reals + ". The operation will be retried on next deployment", - null); + newLoadBalancer = loadBalancer.get().with(instance); + fromState = newLoadBalancer.state(); } + // Always store load balancer so that LoadBalancerExpirer can expire partially provisioned load balancers + db.writeLoadBalancer(newLoadBalancer, fromState); + requireInstance(id, instance); } - private void provision(ApplicationTransaction transaction, ClusterSpec.Id clusterId, NodeList nodes) { - provision(transaction, new LoadBalancerId(transaction.application(), clusterId), nodes, true); + private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, NodeList nodes) { + Instant now = nodeRepository.clock().instant(); + LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster); + Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not active load balancer that was never prepared: " + id); + + Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer); + LoadBalancer.State state = instance.isPresent() ? LoadBalancer.State.active : loadBalancer.get().state(); + LoadBalancer newLoadBalancer = loadBalancer.get().with(instance).with(state, now); + db.writeLoadBalancers(List.of(newLoadBalancer), loadBalancer.get().state(), transaction.nested()); + requireInstance(id, instance); } /** Provision or reconfigure a load balancer instance, if necessary */ - private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, Set<Real> reals, - Optional<LoadBalancer> currentLoadBalancer) { + private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, NodeList nodes, Optional<LoadBalancer> currentLoadBalancer) { + Set<Real> reals = realsOf(nodes); if (hasReals(currentLoadBalancer, reals)) return currentLoadBalancer.get().instance(); - log.log(Level.INFO, () -> "Creating " + id + ", targeting: " + reals); + log.log(Level.INFO, () -> "Provisioning instance for " + id + ", targeting: " + reals); try { return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals), allowEmptyReals(currentLoadBalancer))); @@ -241,7 +231,7 @@ public class LoadBalancerProvisioner { /** Returns real servers for given nodes */ private Set<Real> realsOf(NodeList nodes) { - var reals = new LinkedHashSet<Real>(); + Set<Real> reals = new LinkedHashSet<Real>(); for (var node : nodes) { for (var ip : reachableIpAddresses(node)) { reals.add(new Real(HostName.from(node.hostname()), ip)); @@ -289,6 +279,14 @@ public class LoadBalancerProvisioner { return reachable; } + private static void requireInstance(LoadBalancerId id, Optional<LoadBalancerInstance> instance) { + if (instance.isEmpty()) { + // Signal that load balancer is not ready yet + throw new LoadBalancerServiceException("Could not (re)configure " + id + ". The operation will be retried on next deployment", + null); + } + } + private static ClusterSpec.Id effectiveId(ClusterSpec cluster) { return cluster.combinedId().orElse(cluster.id()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index bdc3bdfd816..16fe5ef241a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -300,6 +300,13 @@ public class LoadBalancerProvisionerTest { assertTrue("Load balancer has instance", loadBalancers.get(0).instance().isPresent()); } + @Test + public void provisioning_load_balancer_for_unsupported_cluster_fails_gracefully() { + tester.loadBalancerService().supportsProvisioning(false); + tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")))); + assertTrue("No load balancer provisioned", tester.nodeRepository().loadBalancers().list(app1).asList().isEmpty()); + } + private void dirtyNodesOf(ApplicationId application) { tester.nodeRepository().nodes().deallocate(tester.nodeRepository().nodes().list().owner(application).asList(), Agent.system, this.getClass().getSimpleName()); } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index d867ae9f211..3b3fdd9bc5c 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -1,13 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "enum_store_dictionary.h" -#include "enumstore.h" #include <vespa/vespalib/btree/btree.hpp> -#include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> -#include <vespa/vespalib/btree/btreenodeallocator.hpp> -#include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/datastore/sharded_hash_map.h> #include <vespa/vespalib/datastore/unique_store_dictionary.hpp> #include <vespa/searchlib/util/bufferwriter.h> @@ -15,7 +10,6 @@ #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.enum_store_dictionary"); -using vespalib::datastore::EntryComparator; using vespalib::datastore::EntryRef; using vespalib::datastore::UniqueStoreAddResult; @@ -25,12 +19,8 @@ using vespalib::btree::BTreeNode; template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove_unused_values(const IndexSet& unused, - const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove_unused_values(const IndexList & unused,const EntryComparator& cmp) { - if (unused.empty()) { - return; - } for (const auto& ref : unused) { this->remove(cmp, ref); } @@ -48,9 +38,9 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::~EnumStoreDictionary() = template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const EntryComparator& cmp) { - IndexSet unused; + IndexList unused; // find unused enums if constexpr (has_btree_dictionary) { @@ -58,19 +48,26 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const _enumStore.free_value_if_unused(iter.getKey(), unused); } } else { - this->_hash_dict.foreach_key([this, &unused](EntryRef ref) { _enumStore.free_value_if_unused(ref, unused); }); + this->_hash_dict.foreach_key([this, &unused](EntryRef ref) { + _enumStore.free_value_if_unused(ref, unused); + }); } remove_unused_values(unused, cmp); } template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const IndexList& to_remove, const EntryComparator& cmp) { - IndexSet unused; + IndexList unused; + + EntryRef prev; for (const auto& index : to_remove) { - _enumStore.free_value_if_unused(index, unused); + assert(prev <= index); + if (index != prev) { + _enumStore.free_value_if_unused(index, unused); + prev = index; + } } remove_unused_values(unused, cmp); } @@ -96,8 +93,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove(const EntryCompar template <typename BTreeDictionaryT, typename HashDictionaryT> bool -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_index(const vespalib::datastore::EntryComparator& cmp, - Index& idx) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_index(const EntryComparator& cmp, Index& idx) const { if constexpr (has_hash_dictionary) { auto find_result = this->_hash_dict.find(cmp, EntryRef()); @@ -118,8 +114,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_index(const vespali template <typename BTreeDictionaryT, typename HashDictionaryT> bool -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_frozen_index(const vespalib::datastore::EntryComparator& cmp, - Index& idx) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_frozen_index(const EntryComparator& cmp, Index& idx) const { if constexpr (has_hash_dictionary) { auto find_result = this->_hash_dict.find(cmp, EntryRef()); @@ -140,7 +135,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_frozen_index(const template <typename BTreeDictionaryT, typename HashDictionaryT> std::vector<IEnumStore::EnumHandle> -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_matching_enums(const EntryComparator& cmp) const { std::vector<IEnumStore::EnumHandle> result; if constexpr (has_btree_dictionary) { @@ -171,14 +166,14 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::get_frozen_root() const template <> std::pair<IEnumStore::Index, EntryRef> -EnumStoreDictionary<EnumTree>::find_posting_list(const vespalib::datastore::EntryComparator&, EntryRef) const +EnumStoreDictionary<EnumTree>::find_posting_list(const EntryComparator&, EntryRef) const { LOG_ABORT("should not be reached"); } template <typename BTreeDictionaryT, typename HashDictionaryT> std::pair<IEnumStore::Index, EntryRef> -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_posting_list(const EntryComparator& cmp, EntryRef root) const { if constexpr (has_hash_dictionary) { (void) root; @@ -199,7 +194,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_posting_list(const template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::collect_folded(Index idx, EntryRef, const std::function<void(vespalib::datastore::EntryRef)>& callback) const +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::collect_folded(Index idx, EntryRef, const std::function<void(EntryRef)>& callback) const { callback(idx); } @@ -244,14 +239,14 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::clear_all_posting_lists( template <> void -EnumStoreDictionary<EnumTree>::update_posting_list(Index, const vespalib::datastore::EntryComparator&, std::function<EntryRef(EntryRef)>) +EnumStoreDictionary<EnumTree>::update_posting_list(Index, const EntryComparator&, std::function<EntryRef(EntryRef)>) { LOG_ABORT("should not be reached"); } template <typename BTreeDictionaryT, typename HashDictionaryT> void -EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) +EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::update_posting_list(Index idx, const EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) { if constexpr (has_btree_dictionary) { auto& dict = this->_btree_dict; @@ -336,7 +331,7 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::get_posting_dictionary() return this->_btree_dict; } -EnumStoreFoldedDictionary::EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<EntryComparator> folded_compare) +EnumStoreFoldedDictionary::EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<EntryComparator> compare, std::unique_ptr<EntryComparator> folded_compare) : EnumStoreDictionary<EnumPostingTree>(enumStore, std::move(compare)), _folded_compare(std::move(folded_compare)) { @@ -389,7 +384,7 @@ EnumStoreFoldedDictionary::remove(const EntryComparator& comp, EntryRef ref) } void -EnumStoreFoldedDictionary::collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const +EnumStoreFoldedDictionary::collect_folded(Index idx, EntryRef root, const std::function<void(EntryRef)>& callback) const { BTreeDictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), _btree_dict.getAllocator()); itr.lower_bound(root, idx, *_folded_compare); @@ -421,6 +416,7 @@ namespace vespalib::btree { using search::IEnumStore; using search::EnumTreeTraits; +using datastore::EntryComparatorWrapper; template class BTreeNodeT<IEnumStore::Index, EnumTreeTraits::INTERNAL_SLOTS>; @@ -456,19 +452,19 @@ class BTreeNodeStore<IEnumStore::Index, uint32_t, NoAggregated, template class BTreeRoot<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRoot<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootT<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootT<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeRootBase<IEnumStore::Index, BTreeNoLeafData, NoAggregated, @@ -494,23 +490,23 @@ class BTreeIteratorBase<IEnumStore::Index, uint32_t, NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS, EnumTreeTraits::PATH_SIZE>; template class BTreeConstIterator<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeConstIterator<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeIterator<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTreeIterator<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTree<IEnumStore::Index, BTreeNoLeafData, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; template class BTree<IEnumStore::Index, uint32_t, NoAggregated, - const vespalib::datastore::EntryComparatorWrapper, EnumTreeTraits>; + const EntryComparatorWrapper, EnumTreeTraits>; } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index a39ff524618..3626fb098d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -18,9 +18,10 @@ protected: using EntryRef = IEnumStoreDictionary::EntryRef; using Index = IEnumStoreDictionary::Index; using BTreeDictionaryType = BTreeDictionaryT; + using EntryComparator = IEnumStoreDictionary::EntryComparator; private: using EnumVector = IEnumStoreDictionary::EnumVector; - using IndexSet = IEnumStoreDictionary::IndexSet; + using IndexList = IEnumStoreDictionary::IndexList; using IndexVector = IEnumStoreDictionary::IndexVector; using ParentUniqueStoreDictionary = vespalib::datastore::UniqueStoreDictionary<BTreeDictionaryT, IEnumStoreDictionary, HashDictionaryT>; using generation_t = IEnumStoreDictionary::generation_t; @@ -30,31 +31,28 @@ protected: private: IEnumStore& _enumStore; - void remove_unused_values(const IndexSet& unused, - const vespalib::datastore::EntryComparator& cmp); + void remove_unused_values(const IndexList& unused, const EntryComparator& cmp); public: - EnumStoreDictionary(IEnumStore& enumStore, std::unique_ptr<vespalib::datastore::EntryComparator> compare); + EnumStoreDictionary(IEnumStore& enumStore, std::unique_ptr<EntryComparator> compare); ~EnumStoreDictionary() override; - void free_unused_values(const vespalib::datastore::EntryComparator& cmp) override; + void free_unused_values(const EntryComparator& cmp) override; + void free_unused_values(const IndexList& to_remove, const EntryComparator& cmp) override; - void free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) override; - - void remove(const vespalib::datastore::EntryComparator& comp, vespalib::datastore::EntryRef ref) override; - bool find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const override; - bool find_frozen_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const override; + void remove(const EntryComparator& comp, EntryRef ref) override; + bool find_index(const EntryComparator& cmp, Index& idx) const override; + bool find_frozen_index(const EntryComparator& cmp, Index& idx) const override; std::vector<attribute::IAttributeVector::EnumHandle> - find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const override; + find_matching_enums(const EntryComparator& cmp) const override; EntryRef get_frozen_root() const override; - std::pair<Index, EntryRef> find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const override; - void collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; + std::pair<Index, EntryRef> find_posting_list(const EntryComparator& cmp, EntryRef root) const override; + void collect_folded(Index idx, EntryRef root, const std::function<void(EntryRef)>& callback) const override; Index remap_index(Index idx) override; void clear_all_posting_lists(std::function<void(EntryRef)> clearer) override; - void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) override; + void update_posting_list(Index idx, const EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) override; bool normalize_posting_lists(std::function<EntryRef(EntryRef)> normalize) override; const EnumPostingTree& get_posting_dictionary() const override; }; @@ -71,14 +69,14 @@ public: class EnumStoreFoldedDictionary : public EnumStoreDictionary<EnumPostingTree> { private: - std::unique_ptr<vespalib::datastore::EntryComparator> _folded_compare; + std::unique_ptr<EntryComparator> _folded_compare; public: - EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare); + EnumStoreFoldedDictionary(IEnumStore& enumStore, std::unique_ptr<EntryComparator> compare, std::unique_ptr<EntryComparator> folded_compare); ~EnumStoreFoldedDictionary() override; - vespalib::datastore::UniqueStoreAddResult add(const vespalib::datastore::EntryComparator& comp, std::function<vespalib::datastore::EntryRef(void)> insertEntry) override; - void remove(const vespalib::datastore::EntryComparator& comp, vespalib::datastore::EntryRef ref) override; - void collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; + vespalib::datastore::UniqueStoreAddResult add(const EntryComparator& comp, std::function<EntryRef(void)> insertEntry) override; + void remove(const EntryComparator& comp, EntryRef ref) override; + void collect_folded(Index idx, EntryRef root, const std::function<void(EntryRef)>& callback) const override; Index remap_index(Index idx) override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 326e0916039..59d77ea0558 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -63,7 +63,7 @@ private: EnumStoreT(const EnumStoreT & rhs) = delete; EnumStoreT & operator=(const EnumStoreT & rhs) = delete; - void free_value_if_unused(Index idx, IndexSet &unused) override; + void free_value_if_unused(Index idx, IndexList &unused) override; const vespalib::datastore::UniqueStoreEntryBase& get_entry_base(Index idx) const { return _store.get_allocator().get_wrapped(idx); @@ -153,7 +153,7 @@ public: class BatchUpdater { private: EnumStoreType& _store; - IndexSet _possibly_unused; + IndexList _possibly_unused; public: BatchUpdater(EnumStoreType& store) @@ -168,11 +168,11 @@ public: auto& entry = _store.get_entry_base(idx); entry.dec_ref_count(); if (entry.get_ref_count() == 0) { - _possibly_unused.insert(idx); + _possibly_unused.push_back(idx); } } void commit() { - _store.free_unused_values(_possibly_unused); + _store.free_unused_values(std::move(_possibly_unused)); } }; @@ -198,7 +198,7 @@ public: Index insert(EntryType value); bool find_index(EntryType value, Index& idx) const; void free_unused_values() override; - void free_unused_values(const IndexSet& to_remove); + void free_unused_values(IndexList to_remove); vespalib::MemoryUsage update_stat() override; std::unique_ptr<EnumIndexRemapper> consider_compact_values(const CompactionStrategy& compaction_strategy) override; std::unique_ptr<EnumIndexRemapper> compact_worst_values(bool compact_memory, bool compact_address_space) override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 9885613f4e3..771da8ffa01 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -30,11 +30,11 @@ make_enum_store_dictionary(IEnumStore &store, bool has_postings, const search::D std::unique_ptr<EntryComparator> folded_compare); template <typename EntryT> -void EnumStoreT<EntryT>::free_value_if_unused(Index idx, IndexSet& unused) +void EnumStoreT<EntryT>::free_value_if_unused(Index idx, IndexList& unused) { const auto& entry = get_entry_base(idx); if (entry.get_ref_count() == 0) { - unused.insert(idx); + unused.push_back(idx); _store.get_allocator().hold(idx); } } @@ -140,7 +140,7 @@ EnumStoreT<EntryT>::BatchUpdater::insert(EntryType value) auto cmp = _store.make_comparator(value); auto result = _store._dict->add(cmp, [this, &value]() -> EntryRef { return _store._store.get_allocator().allocate(value); }); if (result.inserted()) { - _possibly_unused.insert(result.ref()); + _possibly_unused.push_back(result.ref()); } return result.ref(); } @@ -191,8 +191,16 @@ EnumStoreT<EntryT>::free_unused_values() template <typename EntryT> void -EnumStoreT<EntryT>::free_unused_values(const IndexSet& to_remove) +EnumStoreT<EntryT>::free_unused_values(IndexList to_remove) { + struct CompareEnumIndex { + using Index = IEnumStore::Index; + + bool operator()(const Index &lhs, const Index &rhs) const { + return lhs.ref() < rhs.ref(); + } + }; + std::sort(to_remove.begin(), to_remove.end(), CompareEnumIndex()); _dict->free_unused_values(to_remove, get_comparator()); } diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 6d714ec25ba..716609764f4 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -40,22 +40,14 @@ public: using EnumIndexRemapper = vespalib::datastore::UniqueStoreRemapper<InternalIndex>; using Enumerator = vespalib::datastore::UniqueStoreEnumerator<IEnumStore::InternalIndex>; - struct CompareEnumIndex { - using Index = IEnumStore::Index; - - bool operator()(const Index &lhs, const Index &rhs) const { - return lhs.ref() < rhs.ref(); - } - }; - - using IndexSet = std::set<Index, CompareEnumIndex>; + using IndexList = std::vector<Index>; virtual ~IEnumStore() = default; virtual void write_value(BufferWriter& writer, Index idx) const = 0; virtual ssize_t load_unique_values(const void* src, size_t available, IndexVector& idx) = 0; virtual void set_ref_count(Index idx, uint32_t ref_count) = 0; - virtual void free_value_if_unused(Index idx, IndexSet& unused) = 0; + virtual void free_value_if_unused(Index idx, IndexList& unused) = 0; virtual void free_unused_values() = 0; virtual bool is_folded_change(Index idx1, Index idx2) const = 0; virtual IEnumStoreDictionary& get_dictionary() = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index f816177b06c..bef7384b0b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -29,29 +29,29 @@ using EnumPostingTree = vespalib::btree::BTree<IEnumStore::Index, uint32_t, class IEnumStoreDictionary : public vespalib::datastore::IUniqueStoreDictionary { public: using EntryRef = vespalib::datastore::EntryRef; + using EntryComparator = vespalib::datastore::EntryComparator; using EnumVector = IEnumStore::EnumVector; using Index = IEnumStore::Index; - using IndexSet = IEnumStore::IndexSet; + using IndexList = IEnumStore::IndexList; using IndexVector = IEnumStore::IndexVector; using generation_t = vespalib::GenerationHandler::generation_t; public: virtual ~IEnumStoreDictionary() = default; - virtual void free_unused_values(const vespalib::datastore::EntryComparator& cmp) = 0; - virtual void free_unused_values(const IndexSet& to_remove, - const vespalib::datastore::EntryComparator& cmp) = 0; - virtual bool find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const = 0; - virtual bool find_frozen_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const = 0; + virtual void free_unused_values(const EntryComparator& cmp) = 0; + virtual void free_unused_values(const IndexList& to_remove, const EntryComparator& cmp) = 0; + virtual bool find_index(const EntryComparator& cmp, Index& idx) const = 0; + virtual bool find_frozen_index(const EntryComparator& cmp, Index& idx) const = 0; virtual std::vector<attribute::IAttributeVector::EnumHandle> - find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const = 0; + find_matching_enums(const EntryComparator& cmp) const = 0; virtual EntryRef get_frozen_root() const = 0; - virtual std::pair<Index, EntryRef> find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const = 0; - virtual void collect_folded(Index idx, EntryRef root, const std::function<void(vespalib::datastore::EntryRef)>& callback) const = 0; + virtual std::pair<Index, EntryRef> find_posting_list(const EntryComparator& cmp, EntryRef root) const = 0; + virtual void collect_folded(Index idx, EntryRef root, const std::function<void(EntryRef)>& callback) const = 0; virtual Index remap_index(Index idx) = 0; virtual void clear_all_posting_lists(std::function<void(EntryRef)> clearer) = 0; - virtual void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) = 0; + virtual void update_posting_list(Index idx, const EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) = 0; virtual bool normalize_posting_lists(std::function<EntryRef(EntryRef)> normalize) = 0; virtual const EnumPostingTree& get_posting_dictionary() const = 0; }; diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index bf75400b157..a9a94afb763 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -312,9 +312,9 @@ SingleValueEnumAttribute<B>::onShrinkLidSpace() uint32_t default_value_ref_count = this->_enumStore.get_ref_count(default_value_ref); assert(default_value_ref_count >= shrink_docs); this->_enumStore.set_ref_count(default_value_ref, default_value_ref_count - shrink_docs); - IEnumStore::IndexSet possibly_unused; - possibly_unused.insert(default_value_ref); - this->_enumStore.free_unused_values(possibly_unused); + IEnumStore::IndexList possibly_unused; + possibly_unused.push_back(default_value_ref); + this->_enumStore.free_unused_values(std::move(possibly_unused)); } _enumIndices.shrink(committedDocIdLimit); this->setNumDocs(committedDocIdLimit); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 0c9148ad834..34e5fe49f69 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -50,7 +50,7 @@ public class SlobrokMonitorManagerImpl extends AbstractComponent implements Slob private SlobrokMonitorManagerImpl(Transport transport, Supervisor orb, DuperModelManager duperModel) { this(() -> new SlobrokMonitor(orb), transport, duperModel); - orb.useSmallBuffers(); + orb.setDropEmptyBuffers(true); } SlobrokMonitorManagerImpl(Supplier<SlobrokMonitor> slobrokMonitorFactory, Transport transport, DuperModelManager duperModel) { diff --git a/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp b/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp index 543712cc4d2..510b1df2ff3 100644 --- a/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp +++ b/storage/src/vespa/storage/distributor/distributor_total_metrics.cpp @@ -18,16 +18,30 @@ DistributorTotalMetrics::DistributorTotalMetrics(uint32_t num_distributor_stripe DistributorTotalMetrics::~DistributorTotalMetrics() = default; void -DistributorTotalMetrics::aggregate() +DistributorTotalMetrics::aggregate_helper(DistributorMetricSet &total) const { - DistributorMetricSet::reset(); - _bucket_db_updater_metrics.addToPart(*this); + _bucket_db_updater_metrics.addToPart(total); for (auto &stripe_metrics : _stripes_metrics) { - stripe_metrics->addToPart(*this); + stripe_metrics->addToPart(total); } } void +DistributorTotalMetrics::aggregate() +{ + DistributorMetricSet::reset(); + aggregate_helper(*this); +} + +void +DistributorTotalMetrics::addToSnapshot(Metric& m, std::vector<Metric::UP> &ownerList) const +{ + DistributorMetricSet total; + aggregate_helper(total); + total.addToSnapshot(m, ownerList); +} + +void DistributorTotalMetrics::reset() { DistributorMetricSet::reset(); diff --git a/storage/src/vespa/storage/distributor/distributor_total_metrics.h b/storage/src/vespa/storage/distributor/distributor_total_metrics.h index 14116af3d3b..f0457fe64c3 100644 --- a/storage/src/vespa/storage/distributor/distributor_total_metrics.h +++ b/storage/src/vespa/storage/distributor/distributor_total_metrics.h @@ -15,10 +15,12 @@ class DistributorTotalMetrics : public DistributorMetricSet { std::vector<std::shared_ptr<DistributorMetricSet>> _stripes_metrics; DistributorMetricSet _bucket_db_updater_metrics; + void aggregate_helper(DistributorMetricSet &total) const; public: explicit DistributorTotalMetrics(uint32_t num_distributor_stripes); ~DistributorTotalMetrics() override; void aggregate(); + void addToSnapshot(Metric& m, std::vector<Metric::UP> &ownerList) const override; void reset() override; DistributorMetricSet& stripe(uint32_t stripe_index) { return *_stripes_metrics[stripe_index]; } DistributorMetricSet& bucket_db_updater_metrics() { return _bucket_db_updater_metrics; } diff --git a/vespa-feed-client-cli/pom.xml b/vespa-feed-client-cli/pom.xml index 9fd59f1cfa4..ebbea35f4a4 100644 --- a/vespa-feed-client-cli/pom.xml +++ b/vespa-feed-client-cli/pom.xml @@ -71,25 +71,38 @@ </configuration> </plugin> <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <archive> - <manifest> - <mainClass>ai.vespa.feed.client.CliClient</mainClass> - </manifest> - </archive> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - <appendAssemblyId>false</appendAssemblyId> - </configuration> <executions> <execution> - <id>make-assembly</id> + <id>make-fatjar</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <attach>false</attach> + <archive> + <manifest> + <mainClass>ai.vespa.feed.client.CliClient</mainClass> + </manifest> + </archive> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + <execution> + <id>make-zip</id> <phase>package</phase> <goals> <goal>single</goal> </goals> + <configuration> + <descriptors> + <descriptor>src/maven/create-zip.xml</descriptor> + </descriptors> + </configuration> </execution> </executions> </plugin> diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh new file mode 100755 index 00000000000..57077205d18 --- /dev/null +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +exec java \ +-Djava.awt.headless=true \ +-Xms128m -Xmx2048m \ +--add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ +-Djava.util.logging.config.file=logging.properties \ +-cp vespa-feed-client-cli-jar-with-dependencies.jar ai.vespa.feed.client.CliClient "$@" diff --git a/vespa-feed-client-cli/src/maven/create-zip.xml b/vespa-feed-client-cli/src/maven/create-zip.xml new file mode 100644 index 00000000000..45bbbea9f2d --- /dev/null +++ b/vespa-feed-client-cli/src/maven/create-zip.xml @@ -0,0 +1,24 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd"> + <id>zip</id> + <includeBaseDirectory>true</includeBaseDirectory> + + <formats> + <format>zip</format> + </formats> + <files> + <file> + <source>${project.build.directory}/${project.artifactId}-jar-with-dependencies.jar</source> + </file> + <file> + <source>${project.basedir}/src/main/sh/vespa-feed-client-standalone.sh</source> + <destName>vespa-feed-client</destName> + </file> + <file> + <source>${project.basedir}/src/main/resources/logging.properties</source> + </file> + </files> +</assembly> diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/DocumentId.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/DocumentId.java index 21513a5dac2..39fc9fb28e0 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/DocumentId.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/DocumentId.java @@ -8,6 +8,8 @@ import java.util.OptionalLong; import static java.util.Objects.requireNonNull; /** + * Represents a Vespa document id + * * @author jonmv */ public class DocumentId { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java index 39d343515fe..250809a48b9 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java @@ -5,6 +5,8 @@ import java.io.Closeable; import java.util.concurrent.CompletableFuture; /** + * Asynchronous feed client accepting document operations as JSON + * * @author bjorncs * @author jonmv */ diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index df1f3bcd54c..e0418836c80 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -45,8 +45,10 @@ public class FeedClientBuilder { PrivateKey privateKey; Collection<X509Certificate> caCertificates; + /** Creates a builder for a single container endpoint **/ public static FeedClientBuilder create(URI endpoint) { return new FeedClientBuilder(Collections.singletonList(endpoint)); } + /** Creates a builder for multiple container endpoints **/ public static FeedClientBuilder create(List<URI> endpoints) { return new FeedClientBuilder(endpoints); } private FeedClientBuilder(List<URI> endpoints) { @@ -86,61 +88,85 @@ public class FeedClientBuilder { return this; } + /** Sets {@link SSLContext} instance. */ public FeedClientBuilder setSslContext(SSLContext context) { this.sslContext = requireNonNull(context); return this; } + /** Sets {@link HostnameVerifier} instance (e.g for disabling default SSL hostname verification). */ public FeedClientBuilder setHostnameVerifier(HostnameVerifier verifier) { this.hostnameVerifier = requireNonNull(verifier); return this; } + /** Adds HTTP request header to all client requests. */ public FeedClientBuilder addRequestHeader(String name, String value) { return addRequestHeader(name, () -> requireNonNull(value)); } + /** + * Adds HTTP request header to all client requests. Value {@link Supplier} is invoked for each HTTP request, + * i.e. value can be dynamically updated during a feed. + */ public FeedClientBuilder addRequestHeader(String name, Supplier<String> valueSupplier) { this.requestHeaders.put(requireNonNull(name), requireNonNull(valueSupplier)); return this; } + /** + * Overrides default retry strategy. + * @see FeedClient.RetryStrategy + */ public FeedClientBuilder setRetryStrategy(FeedClient.RetryStrategy strategy) { this.retryStrategy = requireNonNull(strategy); return this; } + /** + * Overrides default circuit breaker. + * @see FeedClient.CircuitBreaker + */ public FeedClientBuilder setCircuitBreaker(FeedClient.CircuitBreaker breaker) { this.circuitBreaker = requireNonNull(breaker); return this; } + /** Sets path to client SSL certificate/key PEM files */ public FeedClientBuilder setCertificate(Path certificatePemFile, Path privateKeyPemFile) { this.certificateFile = certificatePemFile; this.privateKeyFile = privateKeyPemFile; return this; } + /** Sets client SSL certificates/key */ public FeedClientBuilder setCertificate(Collection<X509Certificate> certificate, PrivateKey privateKey) { this.certificate = certificate; this.privateKey = privateKey; return this; } + /** Sets client SSL certificate/key */ public FeedClientBuilder setCertificate(X509Certificate certificate, PrivateKey privateKey) { return setCertificate(Collections.singletonList(certificate), privateKey); } + /** + * Overrides JVM default SSL truststore + * @param caCertificatesFile Path to PEM encoded file containing trusted certificates + */ public FeedClientBuilder setCaCertificatesFile(Path caCertificatesFile) { this.caCertificatesFile = caCertificatesFile; return this; } + /** Overrides JVM default SSL truststore */ public FeedClientBuilder setCaCertificates(Collection<X509Certificate> caCertificates) { this.caCertificates = caCertificates; return this; } + /** Constructs instance of {@link ai.vespa.feed.client.FeedClient} from builder configuration */ public FeedClient build() { try { validateConfiguration(); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedException.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedException.java index 25068818396..e1c6c733e9c 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedException.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedException.java @@ -2,6 +2,8 @@ package ai.vespa.feed.client; /** + * Signals that an error occurred during feeding + * * @author bjorncs */ public class FeedException extends RuntimeException { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonParseException.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonParseException.java index 785ffd8eb4c..8edf74ec275 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonParseException.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonParseException.java @@ -2,6 +2,8 @@ package ai.vespa.feed.client; /** + * Signals that supplied JSON is invalid + * * @author bjorncs */ public class JsonParseException extends FeedException { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/OperationParameters.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/OperationParameters.java index 22546f89ccb..8c20a37d224 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/OperationParameters.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/OperationParameters.java @@ -7,6 +7,8 @@ import java.util.Optional; import java.util.OptionalInt; /** + * Per-operation feed parameters + * * @author bjorncs * @author jonmv */ diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/Result.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/Result.java index 31a6cf6e893..b29d65e193b 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/Result.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/Result.java @@ -4,6 +4,8 @@ package ai.vespa.feed.client; import java.util.Optional; /** + * Result for a document operation + * * @author bjorncs * @author jonmv */ diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index 73e10c39419..15b3d2e9d7d 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -57,7 +57,10 @@ public class VespaRecordWriter extends RecordWriter<Object, Object> { if (error instanceof JsonParseException) { counters.incrementDocumentsSkipped(1); } else { - log.warning("Failed to feed single document: " + error); + String msg = "Failed to feed single document: " + error; + System.out.println(msg); + System.err.println(msg); + log.warning(msg); counters.incrementDocumentsFailed(1); } } else { diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index 11b4f5e6631..90281acb0d3 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -141,6 +141,38 @@ assertMemStats(const DataStoreBase::MemStats &exp, EXPECT_EQ(exp._holdBuffers, act._holdBuffers); } +TEST(DataStoreTest, require_that_invalid_entry_ref_can_be_ordered) { + EntryRef inValid; + EntryRef a(1); + EXPECT_EQ(inValid, inValid); + EXPECT_EQ(a, a); + EXPECT_NE(inValid, a); + EXPECT_NE(a, inValid); + EXPECT_LT(inValid, a); + EXPECT_LE(inValid, a); +} + +TEST(DataStoreTest, require_that_entry_ref_can_be_ordered) { + EntryRef a(1); + EntryRef b(2); + EntryRef c(3); + EXPECT_EQ(a, a); + EXPECT_EQ(b, b); + EXPECT_EQ(c, c); + EXPECT_NE(a, b); + EXPECT_NE(a, c); + EXPECT_NE(b, c); + EXPECT_LT(a, b); + EXPECT_LT(b, c); + EXPECT_LT(a, c); + EXPECT_LE(a, a); + EXPECT_LE(b, b); + EXPECT_LE(c, c); + EXPECT_LE(a, b); + EXPECT_LE(b, c); + EXPECT_LE(a, c); +} + TEST(DataStoreTest, require_that_entry_ref_is_working) { using MyRefType = EntryRefT<22>; @@ -643,6 +675,7 @@ TEST(DataStoreTest, control_static_sizes) { EXPECT_EQ(0, bs.size()); } + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.h b/vespalib/src/vespa/vespalib/datastore/entryref.h index 046d9089580..01f473fcf17 100644 --- a/vespalib/src/vespa/vespalib/datastore/entryref.h +++ b/vespalib/src/vespa/vespalib/datastore/entryref.h @@ -21,6 +21,7 @@ public: bool operator==(const EntryRef &rhs) const noexcept { return _ref == rhs._ref; } bool operator!=(const EntryRef &rhs) const noexcept { return _ref != rhs._ref; } bool operator <(const EntryRef &rhs) const noexcept { return _ref < rhs._ref; } + bool operator <=(const EntryRef &rhs) const noexcept { return _ref <= rhs._ref; } }; /** |