diff options
52 files changed, 337 insertions, 383 deletions
diff --git a/client/go/jvm/application_container.go b/client/go/jvm/application_container.go index 1cbc8877aa3..b7c9a88b5fd 100644 --- a/client/go/jvm/application_container.go +++ b/client/go/jvm/application_container.go @@ -45,7 +45,7 @@ func (a *ApplicationContainer) Discriminator() string { func (a *ApplicationContainer) addJdiscProperties() { cfgId := a.ConfigId() - opts := a.jvmArgs + opts := a.jvmOpts opts.AddCommonJdiscProperties() containerParentDir := defaults.UnderVespaHome("var/jdisc_container") containerHomeDir := fmt.Sprintf("%s/%s", containerParentDir, a.Discriminator()) @@ -104,7 +104,7 @@ func (a *ApplicationContainer) configureMemory(qc *QrStartConfig) { jvm_directMemorySizeCache = 0 } maxDirectMemorySize := jvm_baseMaxDirectMemorySize + (jvm_heapsize / 8) + jvm_directMemorySizeCache - opts := a.jvmArgs + opts := a.jvmOpts opts.AddOption(fmt.Sprintf("-Xms%dm", jvm_minHeapsize)) opts.AddOption(fmt.Sprintf("-Xmx%dm", jvm_heapsize)) opts.AddOption(fmt.Sprintf("-XX:ThreadStackSize=%d", jvm_stacksize)) diff --git a/client/go/jvm/container.go b/client/go/jvm/container.go index 53ff9f9a809..fd65602e573 100644 --- a/client/go/jvm/container.go +++ b/client/go/jvm/container.go @@ -25,7 +25,7 @@ type Container interface { type containerBase struct { configId string serviceName string - jvmArgs *Options + jvmOpts *Options propsFile string } @@ -34,7 +34,7 @@ func (cb *containerBase) ServiceName() string { } func (cb *containerBase) JvmOptions() *Options { - return cb.jvmArgs + return cb.jvmOpts } func (cb *containerBase) ConfigId() string { diff --git a/client/go/jvm/options_test.go b/client/go/jvm/options_test.go index cfb4e90d5e5..a2763223611 100644 --- a/client/go/jvm/options_test.go +++ b/client/go/jvm/options_test.go @@ -15,7 +15,7 @@ func (*dummyContainer) exportExtraEnv(ps *prog.Spec) {} func newDummyContainer() Container { var dc dummyContainer dc.serviceName = "foo" - dc.jvmArgs = NewOptions(&dc) + dc.jvmOpts = NewOptions(&dc) return &dc } diff --git a/client/go/jvm/run.go b/client/go/jvm/run.go index d0ce2f72988..eb728cabdea 100644 --- a/client/go/jvm/run.go +++ b/client/go/jvm/run.go @@ -27,7 +27,7 @@ func NewApplicationContainer(extraArgs []string) Container { var a ApplicationContainer a.configId = os.Getenv(envvars.VESPA_CONFIG_ID) a.serviceName = os.Getenv(envvars.VESPA_SERVICE_NAME) - a.jvmArgs = NewOptions(&a) + a.jvmOpts = NewOptions(&a) a.configureOptions() for _, x := range extraArgs { a.JvmOptions().AddOption(x) diff --git a/client/go/jvm/standalone_container.go b/client/go/jvm/standalone_container.go index 33007862f58..06d8f041169 100644 --- a/client/go/jvm/standalone_container.go +++ b/client/go/jvm/standalone_container.go @@ -33,7 +33,7 @@ func (a *StandaloneContainer) ConfigId() string { } func (a *StandaloneContainer) configureOptions() { - opts := a.jvmArgs + opts := a.jvmOpts opts.ConfigureCpuCount(0) opts.AddCommonXX() opts.AddOption("-XX:-OmitStackTraceInFastThrow") @@ -52,7 +52,7 @@ func (a *StandaloneContainer) configureOptions() { func NewStandaloneContainer(svcName string) Container { var a StandaloneContainer a.serviceName = svcName - a.jvmArgs = NewOptions(&a) + a.jvmOpts = NewOptions(&a) a.configureOptions() return &a } diff --git a/client/go/util/fix_fs_test.go b/client/go/util/fix_fs_test.go index 8696359ea19..1f0d317957d 100644 --- a/client/go/util/fix_fs_test.go +++ b/client/go/util/fix_fs_test.go @@ -2,9 +2,9 @@ package util import ( - "fmt" "os" "os/user" + "path/filepath" "strconv" "testing" @@ -13,7 +13,8 @@ import ( ) func setup(t *testing.T) string { - tmpDir := t.TempDir() + tt := t.TempDir() + tmpDir, _ := filepath.EvalSymlinks(tt) err := os.MkdirAll(tmpDir+"/a", 0755) assert.Nil(t, err) err = os.MkdirAll(tmpDir+"/a/bad", 0) @@ -81,7 +82,7 @@ func TestSimpleFixes(t *testing.T) { } func TestSuperUserOnly(t *testing.T) { - trace.AdjustVerbosity(2) + trace.AdjustVerbosity(0) var userId int = -1 var groupId int = -1 if os.Getuid() != 0 { @@ -118,7 +119,7 @@ func TestSuperUserOnly(t *testing.T) { func expectSimplePanic() { if r := recover(); r != nil { if jee, ok := r.(*JustExitError); ok { - fmt.Fprintln(os.Stderr, "got as expected:", jee) + trace.Trace("got as expected:", jee) return } panic(r) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index b81b3524c67..781675b9a30 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.change.CertificateRemovalChangeValidator; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.application.validation.change.CloudAccountChangeValidator; import com.yahoo.vespa.model.application.validation.change.ClusterSizeReductionValidator; @@ -121,7 +122,8 @@ public class Validation { new ContainerRestartValidator(), new NodeResourceChangeValidator(), new RedundancyIncreaseValidator(), - new CloudAccountChangeValidator() + new CloudAccountChangeValidator(), + new CertificateRemovalChangeValidator() }; List<ConfigChangeAction> actions = Arrays.stream(validators) .flatMap(v -> v.validate(currentModel, nextModel, overrides, now).stream()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidator.java index 50fd6572bcc..1df33ab8517 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidator.java @@ -17,7 +17,8 @@ public class CertificateRemovalChangeValidator implements ChangeValidator { current.getContainerClusters() .forEach((clusterId, currentCluster) -> { - validateClients(clusterId, currentCluster.getClients(), next.getContainerClusters().get(clusterId).getClients(), overrides, now); + if(next.getContainerClusters().containsKey(clusterId)) + validateClients(clusterId, currentCluster.getClients(), next.getContainerClusters().get(clusterId).getClients(), overrides, now); }); return List.of(); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/security/NodePrincipal.java b/config-provisioning/src/main/java/com/yahoo/config/provision/security/NodePrincipal.java new file mode 100644 index 00000000000..7e58c9c15ac --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/security/NodePrincipal.java @@ -0,0 +1,51 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision.security; + +import java.security.Principal; +import java.util.Objects; + +/** + * Represents the identity of a hosted Vespa node + * + * @author bjorncs + */ +public class NodePrincipal implements Principal { + + private final NodeIdentity identity; + + public NodePrincipal(NodeIdentity identity) { + this.identity = identity; + } + + public NodeIdentity getIdentity() { + return identity; + } + + @Override + public String getName() { + StringBuilder builder = new StringBuilder(identity.nodeType().name()); + identity.hostname().ifPresent(hostname -> builder.append('/').append(hostname.value())); + identity.applicationId().ifPresent(applicationId -> builder.append('/').append(applicationId.toShortString())); + return builder.toString(); + } + + @Override + public String toString() { + return "NodePrincipal{" + + "identity=" + identity + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodePrincipal that = (NodePrincipal) o; + return Objects.equals(identity, that.identity); + } + + @Override + public int hashCode() { + return Objects.hash(identity); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java index e4f14c7a7b6..f101339ed06 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MockVpcEndpointService.java @@ -4,6 +4,7 @@ import ai.vespa.http.DomainName; import com.yahoo.config.provision.CloudAccount; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import java.util.List; import java.util.Optional; /** @@ -11,13 +12,24 @@ import java.util.Optional; */ public class MockVpcEndpointService implements VpcEndpointService { - public static final VpcEndpointService empty = (name, cluster, account) -> Optional.empty(); + public interface Stub extends VpcEndpointService { + @Override default List<VpcEndpoint> getConnections(ClusterId clusterId, Optional<CloudAccount> account) { + return List.of(new VpcEndpoint("endpoint-1", "available")); + } + } + + public static final Stub empty = (name, cluster, account) -> Optional.empty(); - public VpcEndpointService delegate = empty; + public Stub delegate = empty; @Override public Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account) { return delegate.setPrivateDns(privateDnsName, clusterId, account); } + @Override + public List<VpcEndpoint> getConnections(ClusterId cluster, Optional<CloudAccount> account) { + return List.of(new VpcEndpoint("endpoint-1", "available")); + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java index 109b084f672..5069a429b27 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/VpcEndpointService.java @@ -4,6 +4,7 @@ import ai.vespa.http.DomainName; import com.yahoo.config.provision.CloudAccount; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; +import java.util.List; import java.util.Optional; /** @@ -11,10 +12,16 @@ import java.util.Optional; */ public interface VpcEndpointService { + /** Create a TXT record with this name and token, then run the trigger, to pass this challenge. */ + record DnsChallenge(RecordName name, RecordData data, Runnable trigger) { } + /** Sets the private DNS name for any VPC endpoint for the given cluster, potentially guarded by a challenge. */ Optional<DnsChallenge> setPrivateDns(DomainName privateDnsName, ClusterId clusterId, Optional<CloudAccount> account); - /** Create a TXT record with this name and token, then run the trigger, to pass this challenge. */ - record DnsChallenge(RecordName name, RecordData data, Runnable trigger) { } + /** A connection made to an endpoint service. */ + record VpcEndpoint(String endpointId, String state) { } + + /** Lists all endpoints connected to an endpoint service (owned by account) for the given cluster. */ + List<VpcEndpoint> getConnections(ClusterId cluster, Optional<CloudAccount> account); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 7b9e4530aaa..001c0b8e522 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -55,6 +55,7 @@ import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; @@ -73,6 +74,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.VpcEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -1963,16 +1965,26 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private HttpResponse getPrivateServiceInfo(String tenantName, String applicationName, String instanceName, String environment, String region) { - List<LoadBalancer> lbs = controller.serviceRegistry().configServer().getLoadBalancers(ApplicationId.from(tenantName, applicationName, instanceName), - ZoneId.from(environment, region)); + DeploymentId id = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + ZoneId.from(environment, region)); + List<LoadBalancer> lbs = controller.serviceRegistry().configServer().getLoadBalancers(id.applicationId(), id.zoneId()); Slime slime = new Slime(); Cursor lbArray = slime.setObject().setArray("loadBalancers"); for (LoadBalancer lb : lbs) { Cursor lbObject = lbArray.addObject(); lbObject.setString("cluster", lb.cluster().value()); lb.service().ifPresent(service -> { - lbObject.setString("serviceId", service.id()); + lbObject.setString("serviceId", service.id()); // Really the "serviceName", but this is what the user needs >_< service.allowedUrns().forEach(lbObject.setArray("allowedUrns")::addString); + Cursor endpointsArray = lbObject.setArray("endpoints"); + controller.serviceRegistry().vpcEndpointService() + .getConnections(new ClusterId(id, lb.cluster()), + controller.applications().decideCloudAccountOf(id, controller.applications().requireApplication(TenantAndApplicationId.from(tenantName, applicationName)).deploymentSpec())) + .forEach(endpoint -> { + Cursor endpointObject = endpointsArray.addObject(); + endpointObject.setString("endpointId", endpoint.endpointId()); + endpointObject.setString("state", endpoint.state()); + }); }); } return new SlimeJsonResponse(slime); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 0c9f290c257..40d96f716ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -700,7 +700,8 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET private service info tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/private-service", GET) .userIdentity(USER_ID), - "{\"loadBalancers\":[{\"cluster\":\"default\",\"serviceId\":\"service\",\"allowedUrns\":[\"arne\"]}]}"); + """ + {"loadBalancers":[{"cluster":"default","serviceId":"service","allowedUrns":["arne"],"endpoints":[{"endpointId":"endpoint-1","state":"available"}]}]}"""); // GET service/state/v1 tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/service/storagenode/host.com/state/v1/?foo=bar", GET) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index edf53090e50..da9ea3babe2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -25,7 +25,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService; import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -52,8 +51,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -518,13 +515,12 @@ public class RoutingPoliciesTest { // Challenge answered for endpoint RoutingPoliciesTester tester = new RoutingPoliciesTester(); Map<RecordName, RecordData> challenges = new ConcurrentHashMap<>(); - tester.tester.controllerTester().serviceRegistry().vpcEndpointService().delegate = - (name, cluster, account) -> { - RecordName recordName = RecordName.from("challenge--" + name.value()); - if (challenges.containsKey(recordName)) return Optional.empty(); - RecordData recordData = RecordData.from(account.map(CloudAccount::value).orElse("system")); - return Optional.of(new DnsChallenge(recordName, recordData, () -> challenges.put(recordName, recordData))); - }; + tester.tester.controllerTester().serviceRegistry().vpcEndpointService().delegate = (name, cluster, account) -> { + RecordName recordName = RecordName.from("challenge--" + name.value()); + if (challenges.containsKey(recordName)) return Optional.empty(); + RecordData recordData = RecordData.from(account.map(CloudAccount::value).orElse("system")); + return Optional.of(new DnsChallenge(recordName, recordData, () -> challenges.put(recordName, recordData))); + }; DeploymentContext app = tester.newDeploymentContext("t", "a", "default"); ApplicationPackage appPackage = applicationPackageBuilder().region(zone3.region()).build(); diff --git a/fastos/src/vespa/fastos/thread.cpp b/fastos/src/vespa/fastos/thread.cpp index 7695ce3342e..d172977b222 100644 --- a/fastos/src/vespa/fastos/thread.cpp +++ b/fastos/src/vespa/fastos/thread.cpp @@ -308,7 +308,7 @@ void FastOS_ThreadInterface::Dispatch(FastOS_Runnable *newOwner, void *arg) _owner = newOwner; _startArg = arg; // Set _thread variable before NewThread returns - _owner->_thread = this; + _owner->_thread.store(this, std::memory_order_release); // It is safe to signal after the unlock since _liveCond is still held // so the signalled thread still exists. diff --git a/fastos/src/vespa/fastos/thread.h b/fastos/src/vespa/fastos/thread.h index 95737e9d079..a8cf12825eb 100644 --- a/fastos/src/vespa/fastos/thread.h +++ b/fastos/src/vespa/fastos/thread.h @@ -455,7 +455,7 @@ class FastOS_Runnable { private: friend class FastOS_ThreadInterface; - FastOS_ThreadInterface *_thread; + std::atomic<FastOS_ThreadInterface*> _thread; public: FastOS_Runnable(const FastOS_Runnable&) = delete; @@ -482,9 +482,9 @@ public: */ virtual void Run(FastOS_ThreadInterface *thisThread, void *arguments)=0; - FastOS_ThreadInterface *GetThread() { return _thread; } - const FastOS_ThreadInterface *GetThread() const { return _thread; } - bool HasThread() const { return _thread != nullptr; } + FastOS_ThreadInterface *GetThread() noexcept { return _thread.load(std::memory_order_acquire); } + const FastOS_ThreadInterface *GetThread() const noexcept { return _thread.load(std::memory_order_acquire); } + bool HasThread() const noexcept { return GetThread() != nullptr; } }; #include <vespa/fastos/unix_thread.h> diff --git a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java index 6ee82a6fddb..83110c0021e 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java +++ b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java @@ -212,11 +212,9 @@ public class GramSplitter { /** Substring in code point space */ public UnicodeString substring(int start, int codePoints) { - int offset = s.offsetByCodePoints(start, Math.min(codePoints, s.codePointCount(start, s.length()))); - if (offset < 0) - return new UnicodeString(s.substring(start)); - else - return new UnicodeString(s.substring(start, offset)); + int cps = codePoints * 2 <= s.length() - start ? codePoints + : Math.min(codePoints, s.codePointCount(start, s.length())); + return new UnicodeString(s.substring(start, s.offsetByCodePoints(start, cps))); } /** Returns the position count code points after start (which may be past the end of the string) */ diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java index fa8419e200f..6cefcfbf67a 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java @@ -1,13 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.process; +import com.yahoo.language.process.GramSplitter.Gram; +import com.yahoo.language.process.GramSplitter.GramSplitterIterator; import com.yahoo.language.simple.SimpleLinguistics; import org.junit.Test; import java.util.Arrays; import java.util.Iterator; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author bratseth @@ -168,6 +172,15 @@ public class GramSplitterTestCase { } @Test + public void testLongString() { + String input = "hey ho come 色 let's go, and then we go again!\n色色色".repeat(10_000); + for (GramSplitterIterator grams = new GramSplitter(new CharacterClasses()).split(input, 3); grams.hasNext(); ) { + Gram gram = grams.next(); + gram.extractFrom(input); + } + } + + @Test public void testChineseComma() { String text = "我喜欢红色、蓝色和紫色"; Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2); diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 4c0304fbc71..cab4de7817d 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -14,8 +14,8 @@ com.google.j2objc:j2objc-annotations:1.1 commons-io:commons-io:2.11.0 javax.annotation:javax.annotation-api:1.2 javax.inject:javax.inject:1 -org.apache.commons:commons-compress:1.21 -org.apache.commons:commons-lang3:3.11 +org.apache.commons:commons-compress:1.22 +org.apache.commons:commons-lang3:3.12.0 org.apache.maven:maven-archiver:3.6.0 org.apache.maven:maven-artifact:3.8.6 org.apache.maven:maven-builder-support:3.8.6 diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java index c45d53ae97b..c4c9dd3f591 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java @@ -25,4 +25,8 @@ public interface NodeRepository { void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes); void setNodeState(String hostName, NodeState nodeState); + + default void reboot(String hostname) { + throw new UnsupportedOperationException("Rebooting not supported in " + getClass().getName()); + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 793bae9e2ab..36a4703a415 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -133,6 +133,14 @@ public class RealNodeRepository implements NodeRepository { response.throwOnError("Failed to set node state"); } + @Override + public void reboot(String hostname) { + String uri = "/nodes/v2/command/reboot?hostname=" + hostname; + StandardConfigServerResponse response = configServerApi.post(uri, Optional.empty(), StandardConfigServerResponse.class); + logger.info(response.message); + response.throwOnError("Failed to reboot " + hostname); + } + private static NodeSpec createNodeSpec(NodeRepositoryNode node) { Objects.requireNonNull(node.type, "Unknown node type"); NodeType nodeType = NodeType.valueOf(node.type); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 47975c8354a..37583f00547 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -70,8 +70,7 @@ public class CuratorDatabaseClient { private static final Path firmwareCheckPath = root.append("firmwareCheck"); private static final Path archiveUrisPath = root.append("archiveUris"); - // TODO: Explain reasoning behind timeout value (why its it as high as 10 minutes?) - private static final Duration defaultLockTimeout = Duration.ofMinutes(10); + private static final Duration defaultLockTimeout = Duration.ofMinutes(1); private final NodeSerializer nodeSerializer; private final CuratorDatabase db; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 2c93992dcab..f77b98cc02c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -33,6 +33,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,10 +78,34 @@ public class NodePatcher { * Note: This may patch more than one node if the field being patched must be applied recursively to host and node. */ public void patch(String hostname, InputStream json) { + unifiedPatch(hostname, json, false); + } + + /** Apply given JSON from a tenant host that may have been compromised. */ + public void patchFromUntrustedTenantHost(String hostname, InputStream json) { + unifiedPatch(hostname, json, true); + } + + private void unifiedPatch(String hostname, InputStream json, boolean untrustedTenantHost) { Inspector root = Exceptions.uncheck(() -> SlimeUtils.jsonToSlime(json.readAllBytes())).get(); Map<String, Inspector> fields = new HashMap<>(); root.traverse(fields::put); + if (untrustedTenantHost) { + var disallowedFields = new HashSet<>(fields.keySet()); + disallowedFields.removeAll(Set.of("currentDockerImage", + "currentFirmwareCheck", + "currentOsVersion", + "currentRebootGeneration", + "currentRestartGeneration", + "reports", + "trustStore", + "vespaVersion")); + if (!disallowedFields.isEmpty()) { + throw new IllegalArgumentException("Patching fields not supported: " + disallowedFields); + } + } + // Create views grouping fields by their locking requirements Map<String, Inspector> regularFields = Maps.filterKeys(fields, k -> !IP_CONFIG_FIELDS.contains(k)); Map<String, Inspector> ipConfigFields = Maps.filterKeys(fields, IP_CONFIG_FIELDS::contains); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 2f35d0e7e81..6e80e559b20 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.security.NodePrincipal; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; @@ -173,7 +174,11 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { if (path.matches("/nodes/v2/node/{hostname}")) { NodePatcher patcher = new NodePatcher(nodeFlavors, nodeRepository); String hostname = path.get("hostname"); - patcher.patch(hostname, request.getData()); + if (isTenantPeer(request)) { + patcher.patchFromUntrustedTenantHost(hostname, request.getData()); + } else { + patcher.patch(hostname, request.getData()); + } return new MessageResponse("Updated " + hostname); } else if (path.matches("/nodes/v2/application/{applicationId}")) { @@ -195,6 +200,15 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { throw new NotFoundException("Nothing at '" + path + "'"); } + /** Returns true if the peer is a tenant host or node. */ + private boolean isTenantPeer(HttpRequest request) { + return request.getJDiscRequest().getUserPrincipal() instanceof NodePrincipal nodePrincipal && + switch (nodePrincipal.getIdentity().nodeType()) { + case host, tenant -> true; + default -> false; + }; + } + private HttpResponse handlePOST(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/nodes/v2/command/restart")) { diff --git a/parent/pom.xml b/parent/pom.xml index 01babcb9020..2f61600424d 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -609,7 +609,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> - <version>3.11</version> + <version>3.12.0</version> </dependency> <dependency> <groupId>io.jsonwebtoken</groupId> @@ -627,24 +627,24 @@ <version>${jjwt.version}</version> </dependency> <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + <version>2.12.2</version> + </dependency> + <dependency> <groupId>net.openhft</groupId> <artifactId>zero-allocation-hashing</artifactId> <version>${zero-allocation-hashing.version}</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> - <artifactId>commons-text</artifactId> - <version>${commons-text.version}</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> <artifactId>commons-math3</artifactId> <version>${commons.math3.version}</version> </dependency> <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> - <version>1.4</version> + <version>1.5.0</version> </dependency> <dependency> <groupId>commons-codec</groupId> @@ -652,16 +652,6 @@ <version>${commons-codec.version}</version> </dependency> <dependency> - <groupId>commons-net</groupId> - <artifactId>commons-net</artifactId> - <version>2.0</version> - </dependency> - <dependency> - <groupId>commons-pool</groupId> - <artifactId>commons-pool</artifactId> - <version>1.5.6</version> - </dependency> - <dependency> <groupId>io.airlift</groupId> <artifactId>aircompressor</artifactId> <version>${aircompressor.version}</version> @@ -729,7 +719,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> - <version>1.21</version> + <version>1.22</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> @@ -1126,7 +1116,6 @@ <curator.version>5.3.0</curator.version> <commons-codec.version>1.15</commons-codec.version> <commons-io.version>2.11.0</commons-io.version> - <commons-text.version>1.10.0</commons-text.version> <commons.math3.version>3.6.1</commons.math3.version> <eclipse-collections.version>11.0.0</eclipse-collections.version> <felix.version>7.0.1</felix.version> diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 0e3445d0785..a66785cb567 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -865,6 +865,15 @@ TEST_F("require that shrink flushtarget is handed over to new attribute manager" EXPECT_EQUAL(am1->getShrinker("a1"), am3->getShrinker("a1")); } +TEST_F("transient resource usage is zero in steady state", Fixture) +{ + f.addAttribute("a1"); + f.addAttribute("a2"); + auto usage = f._m.get_transient_resource_usage(); + EXPECT_EQUAL(0u, usage.disk()); + EXPECT_EQUAL(0u, usage.memory()); +} + TEST_MAIN() { std::filesystem::remove_all(std::filesystem::path(test_dir)); diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 12477469e04..43717fc724f 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -749,6 +749,13 @@ TEST_F("require that flush targets can be retrieved", SearchableFixture) EXPECT_TRUE(assertTarget("subdb.summary.shrink", FType::GC, FComponent::DOCUMENT_STORE, *targets[9])); } +TEST_F("transient resource usage is zero in steady state", SearchableFixture) +{ + auto usage = f._subDb.get_transient_resource_usage(); + EXPECT_EQUAL(0u, usage.disk()); + EXPECT_EQUAL(0u, usage.memory()); +} + TEST_F("require that only fast-access attributes are instantiated", FastAccessOnlyFixture) { std::vector<AttributeGuard> attrs; diff --git a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp index 399ebf171af..a45fb7591dc 100644 --- a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/documentmetastore/lid_allocator.h> +#include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/vespalib/util/generationholder.h> @@ -8,7 +9,9 @@ #include <vespa/vespalib/gtest/gtest.h> #include <iostream> +using search::fef::MatchData; using search::queryeval::Blueprint; +using search::queryeval::SearchIterator; using search::queryeval::SimpleResult; using vespalib::GenerationHolder; using vespalib::Timer; @@ -99,19 +102,25 @@ protected: return result; } - SimpleResult get_active_lids_in_search_iterator(uint32_t docid_limit) { + SimpleResult get_active_lids_in_search_iterator(uint32_t docid_limit, bool filter) { auto blueprint = _allocator.createWhiteListBlueprint(); blueprint->setDocIdLimit(docid_limit); - auto iterator = blueprint->createFilterSearch(true, search::queryeval::Blueprint::FilterConstraint::UPPER_BOUND); + std::unique_ptr<SearchIterator> iterator; + MatchData md(MatchData::params()); + if (filter) { + iterator = blueprint->createFilterSearch(true, Blueprint::FilterConstraint::UPPER_BOUND); + } else { + iterator = blueprint->createSearch(md, true); + } SimpleResult res; res.search(*iterator, docid_limit); return res; } - Trinary search_iterator_matches_any(uint32_t docid_limit) { + Trinary filter_search_iterator_matches_any(uint32_t docid_limit) { auto blueprint = _allocator.createWhiteListBlueprint(); blueprint->setDocIdLimit(docid_limit); - auto iterator = blueprint->createFilterSearch(true, search::queryeval::Blueprint::FilterConstraint::UPPER_BOUND); + auto iterator = blueprint->createFilterSearch(true, Blueprint::FilterConstraint::UPPER_BOUND); return iterator->matches_any(); } @@ -146,16 +155,18 @@ TEST_F(LidAllocatorTest, active_lids_are_available_in_search_iterator) { register_lids({ 1, 2, 3, 4 }); activate_lids({ 1, 2, 4 }, true); - EXPECT_EQ(Trinary::Undefined, search_iterator_matches_any(5)); - EXPECT_EQ(SimpleResult({1, 2, 4}), get_active_lids_in_search_iterator(5)); + EXPECT_EQ(Trinary::Undefined, filter_search_iterator_matches_any(5)); + EXPECT_EQ(SimpleResult({1, 2, 4}), get_active_lids_in_search_iterator(5, true)); + EXPECT_EQ(SimpleResult({1, 2, 4}), get_active_lids_in_search_iterator(5, false)); } -TEST_F(LidAllocatorTest, search_iterator_matches_all_when_all_lids_are_active) +TEST_F(LidAllocatorTest, filter_search_iterator_matches_all_when_all_lids_are_active) { register_lids({ 1, 2, 3, 4 }); activate_lids({ 1, 2, 3, 4 }, true); - EXPECT_EQ(Trinary::True, search_iterator_matches_any(5)); - EXPECT_EQ(SimpleResult({1, 2, 3, 4}), get_active_lids_in_search_iterator(5)); + EXPECT_EQ(Trinary::True, filter_search_iterator_matches_any(6)); + EXPECT_EQ(SimpleResult({1, 2, 3, 4, 5}), get_active_lids_in_search_iterator(6, true)); + EXPECT_EQ(SimpleResult({1, 2, 3, 4}), get_active_lids_in_search_iterator(6, false)); } class LidAllocatorPerformanceTest : public LidAllocatorTest, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 939ae196de8..c03ae3b2f1f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -650,4 +650,18 @@ AttributeManager::readable_attribute_vector(const string& name) const return _importedAttributes->get(name); } +TransientResourceUsage +AttributeManager::get_transient_resource_usage() const +{ + // Transient disk usage is measured as the total disk usage of all attribute snapshots + // that are NOT the valid best one. + // Transient memory usage is zero. + TransientResourceUsage result; + for (const auto& elem : _flushables) { + auto usage = elem.second.getFlusher()->get_transient_resource_usage(); + result.merge(usage); + } + return result; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index b74e7e72a0e..65729767dbb 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -186,6 +186,8 @@ public: const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); } std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; + + TransientResourceUsage get_transient_resource_usage() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index 1a0fdcb32aa..de35ab7394f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -58,6 +58,8 @@ public: std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override; + + TransientResourceUsage get_transient_resource_usage() const override { return {}; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index f1b7eac3712..8101b29d98c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -178,6 +178,12 @@ FlushableAttribute::FlushableAttribute(AttributeVectorSP attr, FlushableAttribute::~FlushableAttribute() = default; +TransientResourceUsage +FlushableAttribute::get_transient_resource_usage() const +{ + return _attrDir->get_transient_resource_usage(); +} + IFlushTarget::SerialNum FlushableAttribute::getFlushedSerialNum() const { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h index 39d79372f25..e25422792fc 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h @@ -2,10 +2,9 @@ #pragma once +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcorespi/flush/iflushtarget.h> #include <vespa/searchlib/common/tunefileinfo.h> -#include <vespa/searchcore/proton/common/hw_info.h> - namespace search { class AttributeVector; } @@ -14,8 +13,8 @@ namespace vespalib { class ISequencedTaskExecutor; } namespace proton { - class AttributeDirectory; +class TransientResourceUsage; /** * Implementation of IFlushTarget interface for attribute vectors. @@ -59,11 +58,12 @@ public: vespalib::ISequencedTaskExecutor &attributeFieldWriter, const HwInfo &hwInfo); - virtual - ~FlushableAttribute(); + virtual ~FlushableAttribute(); void setCleanUpAfterFlush(bool cleanUp) { _cleanUpAfterFlush = cleanUp; } + TransientResourceUsage get_transient_resource_usage() const; + // Implements IFlushTarget virtual MemoryGain getApproxMemoryGain() const override; virtual DiskGain getApproxDiskGain() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h index 4b6b8dc687c..ce163827d42 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h @@ -6,6 +6,7 @@ #include "exclusive_attribute_read_accessor.h" #include "i_attribute_factory.h" #include <vespa/searchcommon/attribute/i_attribute_functor.h> +#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> #include <vespa/searchcorespi/flush/iflushtarget.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/common/serialnum.h> @@ -105,6 +106,8 @@ struct IAttributeManager : public search::IAttributeManager virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) = 0; virtual const ImportedAttributesRepo *getImportedAttributes() const = 0; + + virtual TransientResourceUsage get_transient_resource_usage() const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp index 609ee585a6c..a712035e9af 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp @@ -166,9 +166,13 @@ DocumentMetaStoreFlushTarget(const DocumentMetaStore::SP dms, ITlsSyncer &tlsSyn _lastStats.setPathElementsToLog(8); } - DocumentMetaStoreFlushTarget::~DocumentMetaStoreFlushTarget() = default; +TransientResourceUsage +DocumentMetaStoreFlushTarget::get_transient_resource_usage() const +{ + return _dmsDir->get_transient_resource_usage(); +} IFlushTarget::SerialNum DocumentMetaStoreFlushTarget::getFlushedSerialNum() const diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h index 17072d28515..ef9f9299791 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h @@ -10,10 +10,11 @@ namespace search::common { class FileHeaderContext; } namespace proton { -class ITlsSyncer; -class AttributeDiskLayout; class AttributeDirectory; +class AttributeDiskLayout; class DocumentMetaStore; +class ITlsSyncer; +class TransientResourceUsage; /** * Implementation of IFlushTarget interface for document meta store. @@ -54,6 +55,8 @@ public: void setCleanUpAfterFlush(bool cleanUp) { _cleanUpAfterFlush = cleanUp; } + TransientResourceUsage get_transient_resource_usage() const; + MemoryGain getApproxMemoryGain() const override; DiskGain getApproxDiskGain() const override; Time getLastFlushTime() const override; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 1d02ed4f063..5efc639064e 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -188,12 +188,20 @@ private: mutable std::mutex _lock; mutable std::vector<search::fef::TermFieldMatchData *> _matchDataVector; + std::unique_ptr<SearchIterator> create_search_helper(bool strict) const { + auto tfmd = new search::fef::TermFieldMatchData; + { + std::lock_guard<std::mutex> lock(_lock); + _matchDataVector.push_back(tfmd); + } + return search::BitVectorIterator::create(&_activeLids, get_docid_limit(), *tfmd, strict); + } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 0); (void) tfmda; - return createFilterSearch(strict, FilterConstraint::UPPER_BOUND); + return create_search_helper(strict); } public: WhiteListBlueprint(const search::BitVector &activeLids, bool all_lids_active) @@ -212,12 +220,7 @@ public: if (_all_lids_active) { return std::make_unique<FullSearch>(); } - auto tfmd = new search::fef::TermFieldMatchData; - { - std::lock_guard<std::mutex> lock(_lock); - _matchDataVector.push_back(tfmd); - } - return search::BitVectorIterator::create(&_activeLids, get_docid_limit(), *tfmd, strict); + return create_search_helper(strict); } ~WhiteListBlueprint() { diff --git a/searchcore/src/vespa/searchcore/proton/server/ddbstate.h b/searchcore/src/vespa/searchcore/proton/server/ddbstate.h index 5961ac4da62..20c0799c243 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ddbstate.h +++ b/searchcore/src/vespa/searchcore/proton/server/ddbstate.h @@ -49,7 +49,7 @@ private: static std::vector<vespalib::string> _stateNames; static std::vector<vespalib::string> _configStateNames; - void set_state(State state) noexcept { _state.store(state, std::memory_order_relaxed); } + void set_state(State state) noexcept { _state.store(state, std::memory_order_release); } public: DDBState(); @@ -67,7 +67,7 @@ public: bool enterOnlineState(); void enterShutdownState(); void enterDeadState(); - State getState() const noexcept { return _state.load(std::memory_order_relaxed); } + State getState() const noexcept { return _state.load(std::memory_order_acquire); } static vespalib::string getStateString(State state); bool getClosed() const noexcept { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 27a2db27369..e4d432bf4fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -110,11 +110,12 @@ public: explicit DocumentDBResourceUsageProvider(const DocumentDB& doc_db) noexcept : _doc_db(doc_db) {} + TransientResourceUsage get_transient_resource_usage() const override { - // Transient disk usage is measured as the total disk usage of all current fusion indexes. - // Transient memory usage is measured as the total memory usage of all memory indexes. - auto stats = _doc_db.getReadySubDB()->getSearchableStats(); - return {stats.fusion_size_on_disk(), stats.memoryUsage().allocatedBytes()}; + if (!_doc_db.get_state().get_load_done()) { + return {0, 0}; + } + return _doc_db.getReadySubDB()->get_transient_resource_usage(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index ef885113079..29a25069367 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -422,6 +422,7 @@ public: ExecutorThreadingService & getWriteService() { return _writeService; } void set_attribute_usage_listener(std::unique_ptr<IAttributeUsageListener> listener); + const DDBState& get_state() const noexcept { return _state; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index ebe20f24d92..d5475d1f904 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -352,4 +352,12 @@ FastAccessDocSubDB::getNewestFlushedSerial() return highest; } +TransientResourceUsage +FastAccessDocSubDB::get_transient_resource_usage() const +{ + auto result = StoreOnlyDocSubDB::get_transient_resource_usage(); + result.merge(getAttributeManager()->get_transient_resource_usage()); + return result; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index d29c71ea43c..94fca94c75d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -121,6 +121,7 @@ public: SerialNum getOldestFlushedSerial() override; SerialNum getNewestFlushedSerial() override; virtual void pruneRemovedFields(SerialNum serialNum) override; + TransientResourceUsage get_transient_resource_usage() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index f84352a4558..b945c67660b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -28,11 +28,7 @@ class DocumentDBConfig; class DocumentSubDbInitializer; class DocumentSubDbInitializerResult; class FeedHandler; -struct IAttributeManager; -struct IBucketStateCalculator; -struct IDocumentDBReferenceResolver; class IDocumentDBReference; -struct IDocumentMetaStoreContext; class IDocumentRetriever; class IFeedView; class IIndexWriter; @@ -40,9 +36,14 @@ class IReplayConfig; class ISearchHandler; class ISummaryAdapter; class ISummaryManager; +class PendingLidTrackerBase; class ReconfigParams; class RemoveDocumentsOperation; -class PendingLidTrackerBase; +class TransientResourceUsage; +struct IAttributeManager; +struct IBucketStateCalculator; +struct IDocumentDBReferenceResolver; +struct IDocumentMetaStoreContext; /** * Interface for a document sub database that handles a subset of the documents that belong to a @@ -123,6 +124,7 @@ public: virtual void tearDownReferences(IDocumentDBReferenceResolver &resolver) = 0; virtual void validateDocStore(FeedHandler &op, SerialNum serialNum) const = 0; virtual PendingLidTrackerBase & getUncommittedLidsTracker() = 0; + virtual TransientResourceUsage get_transient_resource_usage() const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index b623b461f6e..043e9cd5d3f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -343,4 +343,15 @@ SearchableDocSubDB::clearViews() { Parent::clearViews(); } +TransientResourceUsage +SearchableDocSubDB::get_transient_resource_usage() const +{ + auto result = FastAccessDocSubDB::get_transient_resource_usage(); + // Transient disk usage is measured as the total disk usage of all current fusion indexes. + // Transient memory usage is measured as the total memory usage of all memory indexes. + auto stats = getSearchableStats(); + result.merge({stats.fusion_size_on_disk(), stats.memoryUsage().allocatedBytes()}); + return result; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index d264a625e96..c628d9a96b7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -131,6 +131,7 @@ public: void close() override; std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override; void tearDownReferences(IDocumentDBReferenceResolver &resolver) override; + TransientResourceUsage get_transient_resource_usage() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 032307c1157..9419dfa1c90 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -585,4 +585,10 @@ addTags(vespalib::GenericHeader &header, const vespalib::string &name) const header.putTag(Tag("subDB", _subDB)); } +TransientResourceUsage +StoreOnlyDocSubDB::get_transient_resource_usage() const +{ + return _dmsFlushTarget->get_transient_resource_usage(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index cb1f1ed07bb..f694cc7298f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -236,6 +236,7 @@ public: PendingLidTrackerBase & getUncommittedLidsTracker() override { return *_pendingLidsForCommit; } vespalib::datastore::CompactionStrategy computeCompactionStrategy(vespalib::datastore::CompactionStrategy strategy) const; bool isNodeRetired() const { return _nodeRetired; } + TransientResourceUsage get_transient_resource_usage() const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 6632fbc856a..6c142c97aaf 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -103,6 +103,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB } void tearDownReferences(IDocumentDBReferenceResolver &) override { } + TransientResourceUsage get_transient_resource_usage() const override { return {}; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 75bb3291dd0..987d60dff01 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -105,6 +105,7 @@ public: std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override { return _mock.readable_attribute_vector(name); } + TransientResourceUsage get_transient_resource_usage() const override { return {}; } }; } diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index dafbe695498..030c51b93dc 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -47,7 +47,7 @@ com.yahoo.athenz:athenz-zms-core:1.10.54 com.yahoo.athenz:athenz-zpe-java-client:1.10.54 com.yahoo.athenz:athenz-zts-core:1.10.54 com.yahoo.rdl:rdl-java:1.5.4 -commons-cli:commons-cli:1.4 +commons-cli:commons-cli:1.5.0 commons-codec:commons-codec:1.15 commons-fileupload:commons-fileupload:1.4 commons-io:commons-io:2.11.0 @@ -77,16 +77,16 @@ javax.servlet:javax.servlet-api:3.1.0 javax.validation:validation-api:1.1.0.Final javax.ws.rs:javax.ws.rs-api:2.0.1 javax.xml.bind:jaxb-api:2.3.0 -joda-time:joda-time:2.8.1 +joda-time:joda-time:2.12.2 net.java.dev.jna:jna:5.11.0 net.openhft:zero-allocation-hashing:0.16 org.antlr:antlr-runtime:3.5.2 org.antlr:antlr4-runtime:4.9.3 org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.5 -org.apache.commons:commons-compress:1.21 +org.apache.commons:commons-compress:1.22 org.apache.commons:commons-csv:1.8 org.apache.commons:commons-exec:1.3 -org.apache.commons:commons-lang3:3.11 +org.apache.commons:commons-lang3:3.12.0 org.apache.commons:commons-math3:3.6.1 org.apache.curator:curator-client:5.3.0 org.apache.curator:curator-framework:5.3.0 diff --git a/zookeeper-server/zookeeper-server/pom.xml b/zookeeper-server/zookeeper-server/pom.xml index f6c8952849c..d8dbe35ec9c 100644 --- a/zookeeper-server/zookeeper-server/pom.xml +++ b/zookeeper-server/zookeeper-server/pom.xml @@ -74,11 +74,6 @@ <artifactId>snappy-java</artifactId> <scope>compile</scope> </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> @@ -93,17 +88,6 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <forkMode>once</forkMode> - <systemPropertyVariables> - <zk-version>${zookeeper.version}</zk-version> - </systemPropertyVariables> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> <configuration> <updateReleaseInfo>true</updateReleaseInfo> diff --git a/zookeeper-server/zookeeper-server/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java deleted file mode 100644 index 3a125d72a89..00000000000 --- a/zookeeper-server/zookeeper-server/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeper; - -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.net.HostName; -import com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer; -import com.yahoo.vespa.zookeeper.Reconfigurer; -import com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl; -import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.admin.ZooKeeperAdmin; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.ServerSocket; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.IntStream; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toList; -import static org.junit.Assert.assertEquals; - -/** - * @author jonmv - */ -public class VespaZooKeeperTest { - - static final Path tempDirRoot = getTmpDir(); - static final List<Integer> ports = new ArrayList<>(); - - /** - * Performs dynamic reconfiguration of ZooKeeper servers. - * <p> - * First, a cluster of 3 servers is set up, and some data is written to it. - * Then, 3 new servers are added, and the first 3 marked for retirement; - * this should force the quorum to move the 3 new servers, but not disconnect the old ones. - * Next, the old servers are removed. - * Then, the cluster is reduced to size 1. - * Finally, the cluster grows to size 3 again. - * <p> - * Throughout all of this, quorum should remain, and the data should remain the same. - */ - @Test(timeout = 120_000) - public void testReconfiguration() throws ExecutionException, InterruptedException, IOException, KeeperException, TimeoutException { - List<ZooKeeper> keepers = new ArrayList<>(); - for (int i = 0; i < 8; i++) keepers.add(new ZooKeeper()); - for (int i = 0; i < 8; i++) keepers.get(i).run(); - - // Start the first three servers. - List<ZookeeperServerConfig> configs = getConfigs(0, 0, 3, 0); - for (int i = 0; i < 3; i++) keepers.get(i).config = configs.get(i); - for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Wait for all servers to be up and running. - for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Write data to verify later. - String path = writeData(configs.get(0)); - - // Let three new servers join, causing the three older ones to retire and leave the ensemble. - configs = getConfigs(0, 3, 3, 3); - for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); - // The existing servers can't reconfigure and leave before the joiners are up. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Wait for new quorum to be established. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Verify written data is preserved. - verifyData(path, configs.get(3)); - - // Old servers are removed. - configs = getConfigs(3, 0, 3, 0); - for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); - // Old servers shut down, while the newer servers remain. - for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // Ensure old servers shut down properly. - for (int i = 0; i < 3; i++) keepers.get(i).await(); - // Ensure new servers have reconfigured. - for (int i = 3; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - - // Verify written data is preserved. - verifyData(path, configs.get(3)); - - - // Cluster shrinks to a single server. - configs = getConfigs(5, 0, 1, 0); - for (int i = 3; i < 6; i++) keepers.get(i).config = configs.get(i); - for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // We let the remaining server reconfigure the others out before they die. - for (int i = 3; i < 5; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 3; i < 5; i++) keepers.get(i).await(); - verifyData(path, configs.get(5)); - - // Cluster grows to 3 servers again. - configs = getConfigs(5, 0, 3, 2); - for (int i = 5; i < 8; i++) keepers.get(i).config = configs.get(i); - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - // Wait for the joiners. - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - verifyData(path, configs.get(7)); - - // Let the remaining servers terminate. - for (int i = 5; i < 8; i++) keepers.get(i).config = null; - for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); - for (int i = 5; i < 8; i++) keepers.get(i).await(); - } - - static String writeData(ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { - try (ZooKeeperAdmin admin = createAdmin(config)) { - List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - String node = admin.create("/test-node", "hi".getBytes(UTF_8), acl, CreateMode.PERSISTENT_SEQUENTIAL); - String read = new String(admin.getData(node, false, new Stat()), UTF_8); - assertEquals("hi", read); - return node; - } - } - - static void verifyData(String path, ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { - for (int i = 0; i < 10; i++) { - try (ZooKeeperAdmin admin = createAdmin(config)) { - assertEquals("hi", new String(admin.getData(path, false, new Stat()), UTF_8)); - return; - } - catch (KeeperException.ConnectionLossException e) { - e.printStackTrace(); - Thread.sleep(10 << i); - } - } - } - - static ZooKeeperAdmin createAdmin(ZookeeperServerConfig config) throws IOException { - return new ZooKeeperAdmin(HostName.getLocalhost() + ":" + config.clientPort(), - 10_000, - System.err::println, - new ZkClientConfigBuilder().toConfig()); - } - - static class ZooKeeper { - - final ExecutorService executor = Executors.newSingleThreadExecutor(); - final Phaser phaser = new Phaser(2); - final AtomicReference<Future<?>> future = new AtomicReference<>(); - ZookeeperServerConfig config; - - void run() { - future.set(executor.submit(() -> { - Reconfigurer reconfigurer = new Reconfigurer(new VespaZooKeeperAdminImpl()); - phaser.arriveAndAwaitAdvance(); - while (config != null) { - new ReconfigurableVespaZooKeeperServer(reconfigurer, config); - phaser.arriveAndAwaitAdvance(); // server is now up, let test thread sync here - phaser.arriveAndAwaitAdvance(); // wait before reconfig/teardown to let test thread do stuff - } - reconfigurer.deconstruct(); - })); - } - - void await() throws ExecutionException, InterruptedException, TimeoutException { - future.get().get(30, SECONDS); - } - } - - static List<ZookeeperServerConfig> getConfigs(int removed, int retired, int active, int joining) { - return IntStream.rangeClosed(1, removed + retired + active) - .mapToObj(id -> getConfig(removed, retired, active, joining, id)) - .collect(toList()); - } - - // Config for server #id among retired + active servers, of which the last may be joining, and with offset removed. - static ZookeeperServerConfig getConfig(int removed, int retired, int active, int joining, int id) { - if (id <= removed) - return null; - - Path tempDir = tempDirRoot.resolve("zookeeper-" + id); - return new ZookeeperServerConfig.Builder() - .clientPort(getPorts(id).get(0)) - .dataDir(tempDir.toString()) - .zooKeeperConfigFile(tempDir.resolve("zookeeper.cfg").toString()) - .myid(id) - .myidFile(tempDir.resolve("myid").toString()) - .dynamicReconfiguration(true) - .server(IntStream.rangeClosed(removed + 1, removed + retired + active) - .mapToObj(i -> new ZookeeperServerConfig.Server.Builder() - .id(i) - .clientPort(getPorts(i).get(0)) - .electionPort(getPorts(i).get(1)) - .quorumPort(getPorts(i).get(2)) - .hostname("localhost") - .joining(i - removed > retired + active - joining) - .retired(i - removed <= retired)) - .collect(toList())) - .build(); - } - - static List<Integer> getPorts(int id) { - if (ports.size() < id * 3) { - int previousPort; - if (ports.isEmpty()) { - String[] version = System.getProperty("zk-version").split("\\."); - int versionPortOffset = 0; - for (String part : version) - versionPortOffset = 32 * (versionPortOffset + Integer.parseInt(part)); - previousPort = 20000 + versionPortOffset % 30000; - } - else - previousPort = ports.get(ports.size() - 1); - - for (int i = 0; i < 3; i++) - ports.add(previousPort = nextPort(previousPort)); - } - return ports.subList(id * 3 - 3, id * 3); - } - - static int nextPort(int previousPort) { - for (int j = 1; j <= 30000; j++) { - int port = (previousPort + j); - while (port > 50000) - port -= 30000; - - try (ServerSocket socket = new ServerSocket(port)) { - return socket.getLocalPort(); - } - catch (IOException e) { - System.err.println("Could not bind port " + port + ": " + e); - } - } - throw new RuntimeException("No free ports"); - } - - static Path getTmpDir() { - try { - Path tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "vespa-zk-test"); - tempDir.toFile().deleteOnExit(); - return tempDir.toAbsolutePath(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} |