diff options
85 files changed, 810 insertions, 1724 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java index b499debcc47..25c4cbb2281 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceIdentity.java @@ -7,7 +7,7 @@ import java.util.Optional; /** * A signed instance identity object that includes a client certificate. This is the result of a successful - * {@link InstanceRegistration}. + * {@link InstanceRegistration} and is the same type as InstanceIdentity in the ZTS API. * * @author mpolden */ diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java new file mode 100644 index 00000000000..fbcda5e68cb --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRefresh.java @@ -0,0 +1,40 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.ca.instance; + +import com.yahoo.security.Pkcs10Csr; + +import java.util.Objects; + +/** + * Information for refreshing a instance in the system. This is the same type as InstanceRefreshInformation type in + * the ZTS API. + * + * @author mpolden + */ +public class InstanceRefresh { + + private final Pkcs10Csr csr; + + public InstanceRefresh(Pkcs10Csr csr) { + this.csr = Objects.requireNonNull(csr, "csr must be non-null"); + } + + /** The Certificate Signed Request describing the wanted certificate */ + public Pkcs10Csr csr() { + return csr; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InstanceRefresh that = (InstanceRefresh) o; + return csr.equals(that.csr); + } + + @Override + public int hashCode() { + return Objects.hash(csr); + } + +} diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java index 7a9ec74e075..2a2b702d21b 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java @@ -6,8 +6,8 @@ import com.yahoo.security.Pkcs10Csr; import java.util.Objects; /** - * Information for registering a new instance in the system. This is similar to the InstanceRegisterInformation type in - * ZTS. + * Information for registering a new instance in the system. This is the same type as InstanceRegisterInformation type + * in the ZTS API. * * @author mpolden */ @@ -47,6 +47,7 @@ public class InstanceRegistration { return attestationData; } + /** The Certificate Signed Request describing the wanted certificate */ public Pkcs10Csr csr() { return csr; } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index ba4f0ce932c..b2120f24160 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -25,12 +25,14 @@ import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Clock; import java.util.Optional; +import java.util.function.Function; import java.util.logging.Level; /** * REST API for issuing and refreshing node certificates in a hosted Vespa system. * * The API implements the following subset of methods from the Athenz ZTS REST API: + * * - Instance registration * - Instance refresh * @@ -59,12 +61,12 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { try { switch (request.getMethod()) { case POST: return handlePost(request); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + default: return ErrorResponse.methodNotAllowed("Method " + request.getMethod() + " is unsupported"); } } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + return ErrorResponse.badRequest(request.getMethod() + " " + request.getUri() + " failed: " + Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + log.log(Level.WARNING, "Unexpected error handling " + request.getMethod() + " " + request.getUri(), e); return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); } } @@ -72,13 +74,12 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { private HttpResponse handlePost(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/ca/v1/instance/")) return registerInstance(request); - // TODO: Implement refresh + if (path.matches("/ca/v1/instance/{provider}/{domain}/{service}/{instanceId}")) return refreshInstance(request, path.get("provider"), path.get("service"), path.get("instanceId")); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse registerInstance(HttpRequest request) { - var body = slimeFromRequest(request); - var instanceRegistration = InstanceSerializer.registrationFromSlime(body); + var instanceRegistration = deserializeRequest(request, InstanceSerializer::registrationFromSlime); var certificate = certificates.create(instanceRegistration.csr(), caCertificate(), caPrivateKey()); var instanceId = Certificates.extractDnsName(instanceRegistration.csr()); var identity = new InstanceIdentity(instanceRegistration.provider(), instanceRegistration.service(), instanceId, @@ -86,6 +87,18 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(InstanceSerializer.identityToSlime(identity)); } + private HttpResponse refreshInstance(HttpRequest request, String provider, String service, String instanceId) { + var instanceRefresh = deserializeRequest(request, InstanceSerializer::refreshFromSlime); + var instanceIdFromCsr = Certificates.extractDnsName(instanceRefresh.csr()); + if (!instanceIdFromCsr.equals(instanceId)) { + throw new IllegalArgumentException("Mismatched instance ID and SAN DNS name [instanceId=" + instanceId + + ",dnsName=" + instanceIdFromCsr + "]"); + } + var certificate = certificates.create(instanceRefresh.csr(), caCertificate(), caPrivateKey()); + var identity = new InstanceIdentity(provider, service, instanceIdFromCsr, Optional.of(certificate)); + return new SlimeJsonResponse(InstanceSerializer.identityToSlime(identity)); + } + /** Returns CA certificate from secret store */ private X509Certificate caCertificate() { var keyName = String.format("vespa.external.%s.configserver.ca.cert.cert", system.value().toLowerCase()); @@ -98,9 +111,10 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { return KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(keyName)); } - private static Slime slimeFromRequest(HttpRequest request) { + private static <T> T deserializeRequest(HttpRequest request, Function<Slime, T> serializer) { try { - return SlimeUtils.jsonToSlime(request.getData().readAllBytes()); + var slime = SlimeUtils.jsonToSlime(request.getData().readAllBytes()); + return serializer.apply(slime); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java index 46a09e9c6f2..a2537cd68f1 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java @@ -6,6 +6,7 @@ import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; +import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; /** @@ -33,6 +34,11 @@ public class InstanceSerializer { Pkcs10CsrUtils.fromPem(requireField(CSR_FIELD, root).asString())); } + public static InstanceRefresh refreshFromSlime(Slime slime) { + Cursor root = slime.get(); + return new InstanceRefresh(Pkcs10CsrUtils.fromPem(requireField(CSR_FIELD, root).asString())); + } + public static Slime identityToSlime(InstanceIdentity identity) { Slime slime = new Slime(); Cursor root = slime.setObject(); diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java index 4393c3a25b9..a1d708a1107 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java @@ -35,9 +35,9 @@ public class CertificateAuthorityApiTest extends ContainerTester { public void register_instance() throws Exception { // POST instance registration var csr = CertificateTester.createCsr("node1.example.com"); - assertRegistration(new Request("http://localhost:12345/ca/v1/instance/", - instanceRegistrationJson(csr), - Request.Method.POST)); + assertIdentityResponse(new Request("http://localhost:12345/ca/v1/instance/", + instanceRegistrationJson(csr), + Request.Method.POST)); // POST instance registration with ZTS client var ztsClient = new DefaultZtsClient(URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); @@ -49,9 +49,26 @@ public class CertificateAuthorityApiTest extends ContainerTester { } @Test - public void invalid_register_instance() { + public void refresh_instance() throws Exception { + // POST instance refresh + var csr = CertificateTester.createCsr("node1.example.com"); + assertIdentityResponse(new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/node1.example.com", + instanceRefreshJson(csr), + Request.Method.POST)); + + // POST instance refresh with ZTS client + var ztsClient = new DefaultZtsClient(URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); + var instanceIdentity = ztsClient.refreshInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), + new AthenzService("vespa.external", "tenant"), + "node1.example.com", + csr); + assertEquals("CN=Vespa CA", instanceIdentity.certificate().getIssuerX500Principal().getName()); + } + + @Test + public void invalid_requests() { // POST instance registration with missing fields - assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Missing required field 'provider'\"}", + assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/ failed: Missing required field 'provider'\"}", new Request("http://localhost:12345/ca/v1/instance/", new byte[0], Request.Method.POST)); @@ -61,7 +78,20 @@ public class CertificateAuthorityApiTest extends ContainerTester { var request = new Request("http://localhost:12345/ca/v1/instance/", instanceRegistrationJson(csr), Request.Method.POST); - assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"DNS name not found in CSR\"}", request); + assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/ failed: DNS name not found in CSR\"}", request); + + // POST instance refresh with missing field + assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/node1.example.com failed: Missing required field 'csr'\"}", + new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/node1.example.com", + new byte[0], + Request.Method.POST)); + + // POST instance refresh where instanceId does not match CSR dnsName + csr = CertificateTester.createCsr("node1.example.com"); + assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/node2.example.com failed: Mismatched instance ID and SAN DNS name [instanceId=node2.example.com,dnsName=node1.example.com]\"}", + new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/node2.example.com", + instanceRefreshJson(csr), + Request.Method.POST)); } private void setCaCertificateAndKey() { @@ -72,11 +102,11 @@ public class CertificateAuthorityApiTest extends ContainerTester { .setSecret("vespa.external.main.configserver.ca.key.key", privateKeyPem); } - private void assertRegistration(Request request) { + private void assertIdentityResponse(Request request) { assertResponse(200, (body) -> { var slime = SlimeUtils.jsonToSlime(body); var root = slime.get(); - assertEquals("provider_prod_us-north-1", root.field("provider").asString()); + assertEquals("vespa.external.provider_prod_us-north-1", root.field("provider").asString()); assertEquals("tenant", root.field("service").asString()); assertEquals("node1.example.com", root.field("instanceId").asString()); var pemEncodedCertificate = root.field("x509Certificate").asString(); @@ -86,10 +116,16 @@ public class CertificateAuthorityApiTest extends ContainerTester { }, request); } + private static byte[] instanceRefreshJson(Pkcs10Csr csr) { + var csrPem = Pkcs10CsrUtils.toPem(csr); + var json = "{\"csr\": \"" + csrPem + "\"}"; + return json.getBytes(StandardCharsets.UTF_8); + } + private static byte[] instanceRegistrationJson(Pkcs10Csr csr) { var csrPem = Pkcs10CsrUtils.toPem(csr); var json = "{\n" + - " \"provider\": \"provider_prod_us-north-1\",\n" + + " \"provider\": \"vespa.external.provider_prod_us-north-1\",\n" + " \"domain\": \"vespa.external\",\n" + " \"service\": \"tenant\",\n" + " \"attestationData\": \"identity document generated by config server\",\n" + diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java index 51010422b6d..83ea9249ad0 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java @@ -7,6 +7,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.ca.CertificateTester; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; +import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; import org.junit.Test; @@ -55,6 +56,16 @@ public class InstanceSerializerTest { assertEquals(json, asJsonString(InstanceSerializer.identityToSlime(identity))); } + @Test + public void serialize_instance_refresh() { + var csr = CertificateTester.createCsr(); + var csrPem = Pkcs10CsrUtils.toPem(csr); + var json = "{\"csr\": \"" + csrPem + "\"}"; + var instanceRefresh = new InstanceRefresh(csr); + var deserialized = InstanceSerializer.refreshFromSlime(SlimeUtils.jsonToSlime(json)); + assertEquals(instanceRefresh, deserialized); + } + private static String asJsonString(Slime slime) { try { return new String(SlimeUtils.toJsonBytes(slime), StandardCharsets.UTF_8); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 8addd6d3b6c..e6fef42d67e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -53,9 +53,10 @@ public interface ModelContext { Set<ContainerEndpoint> endpoints(); boolean isBootstrap(); boolean isFirstTimeDeployment(); - boolean useDedicatedNodeForLogserver(); default boolean useFdispatchByDefault() { return false; } // TODO Remove once 7.111 is the oldest old config models default boolean dispatchWithProtobuf() { return true; } // TODO Remove once 7.111 is the oldest old config models + // TODO: Remove when Vespa 7.112 is the oldest config model in use + default boolean useDedicatedNodeForLogserver() { return true; } boolean useAdaptiveDispatch(); // TODO: Remove temporary default implementation default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index a4f2ecd6675..ee4879f2203 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; -import com.yahoo.config.model.ConfigModel; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.XmlHelper; @@ -181,7 +180,7 @@ public class VespaDomBuilder extends VespaModelBuilder { if (port > 0) { t.setBasePort(port); } - allocateHost(t, hostSystem, producerSpec, deployState.getDeployLogger()); + allocateHost(t, hostSystem, producerSpec); } // This depends on which constructor in AbstractService is used, but the best way // is to let this method do initialize. @@ -199,7 +198,7 @@ public class VespaDomBuilder extends VespaModelBuilder { * @param producerSpec xml element for the service */ private void allocateHost(final AbstractService service, HostSystem hostSystem, - Element producerSpec, DeployLogger deployLogger) + Element producerSpec) { // TODO store service on something else than HostSystem, to not make that overloaded service.setHostResource(hostSystem.getHost(producerSpec.getAttribute("hostalias"))); @@ -207,15 +206,15 @@ public class VespaDomBuilder extends VespaModelBuilder { } /** - * The SimpleConfigProducer is the producer for elements such as qrservers, topleveldispatchers, gateways. + * The SimpleConfigProducer is the producer for elements such as qrservers, gateways. * Must support overrides for that too, hence this builder * * @author vegardh */ - public static class DomSimpleConfigProducerBuilder extends DomConfigProducerBuilder<SimpleConfigProducer> { - private String configId = null; + static class DomSimpleConfigProducerBuilder extends DomConfigProducerBuilder<SimpleConfigProducer> { + private String configId; - public DomSimpleConfigProducerBuilder(String configId) { + DomSimpleConfigProducerBuilder(String configId) { this.configId = configId; } @@ -231,7 +230,7 @@ public class VespaDomBuilder extends VespaModelBuilder { /** * @param name The name of the Vespa to create. Usually 'root' when there is only one. */ - public DomRootBuilder(String name) { + DomRootBuilder(String name) { this.name = name; } @@ -249,23 +248,13 @@ public class VespaDomBuilder extends VespaModelBuilder { } /** - * Gets the index from a service's spec - * - * @param spec The element containing the xml specification for this Service. - * @return the index of the service, which is retrieved from the xml spec. - */ - static int getIndex(Element spec) { - return getXmlIntegerAttribute(spec, "index"); - } - - /** * Gets an integer attribute value from a service's spec * * @param spec XML element * @param attributeName nam of attribute to get value from * @return value of attribute, or 0 if it does not exist or is empty */ - static int getXmlIntegerAttribute(Element spec, String attributeName) { + private static int getXmlIntegerAttribute(Element spec, String attributeName) { String value = (spec == null) ? null : spec.getAttribute(attributeName); if (value == null || value.equals("")) { return 0; @@ -287,7 +276,6 @@ public class VespaDomBuilder extends VespaModelBuilder { * @param configModelRepo a {@link ConfigModelRepo} */ public void postProc(DeployLogger deployLogger, AbstractConfigProducer root, ConfigModelRepo configModelRepo) { - createTlds(deployLogger, configModelRepo); setContentSearchClusterIndexes(configModelRepo); createDocprocMBusServersAndClients(configModelRepo); } @@ -303,14 +291,6 @@ public class VespaDomBuilder extends VespaModelBuilder { docproc.getChains().addServersAndClientsForChains(); } - private void createTlds(DeployLogger deployLogger, ConfigModelRepo pc) { - for (ConfigModel p : pc.asMap().values()) { - if (p instanceof Content) { - ((Content)p).createTlds(deployLogger, pc); - } - } - } - /** * For some reason, search clusters need to be enumerated. * @param configModelRepo a {@link ConfigModelRepo} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 84e8cdf144f..5266fc46f6e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.container.component.ContainerSubsystem; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.Dispatch; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.StreamingSearchCluster; @@ -124,13 +123,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> } scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); - if (sys instanceof IndexedSearchCluster) { - for (Dispatch tld: ((IndexedSearchCluster)sys).getTLDs()) { - scB.dispatcher(new QrSearchersConfig.Searchcluster.Dispatcher.Builder(). - host(tld.getHostname()). - port(tld.getDispatchPort())); - } - } else { + if ( ! (sys instanceof IndexedSearchCluster)) { scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder(). routespec(((StreamingSearchCluster)sys).getStorageRouteSpec())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 74caf2d8026..8533c8d430f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -81,23 +81,6 @@ public class Content extends ConfigModel { */ public Optional<ApplicationContainerCluster> ownedIndexingCluster() { return ownedIndexingCluster; } - public void createTlds(DeployLogger deployLogger, ConfigModelRepo modelRepo) { - IndexedSearchCluster indexedCluster = cluster.getSearch().getIndexed(); - if (indexedCluster == null) return; - - SimpleConfigProducer tldParent = new SimpleConfigProducer(indexedCluster, "tlds"); - for (ConfigModel model : modelRepo.asMap().values()) { - if ( ! (model instanceof ContainerModel)) continue; - - ContainerCluster<? extends Container> containerCluster = ((ContainerModel) model).getCluster(); - if (containerCluster.getSearch() == null) continue; // this is not a qrs cluster - - log.log(LogLevel.DEBUG, "Adding tlds for indexed cluster " + indexedCluster.getClusterName() + ", container cluster " + containerCluster.getName()); - indexedCluster.addTldsWithSameIdsAsContainers(deployLogger, tldParent, containerCluster); - } - indexedCluster.setupDispatchGroups(deployLogger); - } - private static boolean containsIndexingChain(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) { if (IndexingDocprocChain.NAME.equals(chainSpec.componentId.stringValue())) return true; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index fc79a3f4bbf..fdf88124012 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -186,11 +186,6 @@ public class ContentCluster extends AbstractConfigProducer implements } index.setSearchCoverage(DomSearchCoverageBuilder.build(element)); index.setDispatchSpec(DomDispatchBuilder.build(element)); - if (index.useMultilevelDispatchSetup()) { - // We must validate this before we add tlds and setup the dispatch groups. - // This must therefore happen before the regular validate() step. - new MultilevelDispatchValidator(index.getClusterName(), index.getDispatchSpec(), index.getSearchNodes()).validate(); - } // TODO: This should be cleaned up to avoid having to change code in 100 places // every time we add a dispatch option. @@ -656,9 +651,6 @@ public class ContentCluster extends AbstractConfigProducer implements if (search.usesHierarchicDistribution() && !isHosted) { // validate manually configured groups new IndexedHierarchicDistributionValidator(search.getClusterName(), rootGroup, redundancy, search.getIndexed().getTuning().dispatch.policy).validate(); - if (search.getIndexed().useMultilevelDispatchSetup()) { - throw new IllegalArgumentException("In indexed content cluster '" + search.getClusterName() + "': Using multi-level dispatch setup is not supported when using hierarchical distribution."); - } } new ReservedDocumentTypeNameValidator().validate(documentDefinitions); new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java deleted file mode 100644 index 2e821a67cb2..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.vespa.config.search.core.FdispatchrcConfig; -import com.yahoo.vespa.config.search.core.PartitionsConfig; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.PortAllocBridge; -import com.yahoo.vespa.model.application.validation.RestartConfigs; -import com.yahoo.vespa.model.content.SearchCoverage; - -import java.util.ArrayList; -import java.util.List; - -/** - * Represents a dispatch (top-level (tld) or mid-level). - * There must be one or more tld instances in a search cluster. - * - * @author arnej27959 - */ -@RestartConfigs({FdispatchrcConfig.class, PartitionsConfig.class}) -public class Dispatch extends AbstractService implements SearchInterface, - FdispatchrcConfig.Producer, - PartitionsConfig.Producer { - - private static final String TLD_NAME = "tld"; - private static final String DISPATCH_NAME = "dispatch"; - - private static final long serialVersionUID = 1L; - private final DispatchGroup dispatchGroup; - private final NodeSpec nodeSpec; - private final int dispatchLevel; - private final boolean preferLocalRow; - private final boolean isTopLevel; - private boolean useAdaptiveDispatch; - - private Dispatch(DispatchGroup dispatchGroup, AbstractConfigProducer parent, String subConfigId, - NodeSpec nodeSpec, int dispatchLevel, boolean preferLocalRow, boolean isTopLevel) { - super(parent, subConfigId); - this.dispatchGroup = dispatchGroup; - this.nodeSpec = nodeSpec; - this.dispatchLevel = dispatchLevel; - this.preferLocalRow = preferLocalRow; - this.isTopLevel = isTopLevel; - this.useAdaptiveDispatch = false; - portsMeta.on(0).tag("rpc").tag("admin"); - portsMeta.on(1).tag("fs4"); - portsMeta.on(2).tag("http").tag("json").tag("health").tag("state"); - setProp("clustertype", "search") - .setProp("clustername", dispatchGroup.getClusterName()) - .setProp("index", nodeSpec.groupIndex()); - } - - public Dispatch useAdaptiveDispatch(boolean useAdaptiveDispatch) { - this.useAdaptiveDispatch = useAdaptiveDispatch; - return this; - } - - public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId) { - return createTld(dispatchGroup, parent, rowId, false); - } - - public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId, boolean preferLocalRow) { - String subConfigId = TLD_NAME + "." + rowId; - return new Dispatch(dispatchGroup, parent, subConfigId, new NodeSpec(rowId, 0), 0, preferLocalRow, true); - } - - public static Dispatch createTldWithContainerIdInName(DispatchGroup dispatchGroup, AbstractConfigProducer parent, String containerName, int containerIndex) { - String subConfigId = containerName + "." + containerIndex + "." + TLD_NAME + "." + containerIndex; - return new Dispatch(dispatchGroup, parent, subConfigId, new NodeSpec(containerIndex, 0), 0, false, true); - } - - public static Dispatch createDispatchWithStableConfigId(DispatchGroup dispatchGroup, AbstractConfigProducer parent, NodeSpec nodeSpec, int distributionKey, int dispatchLevel) { - String subConfigId = DISPATCH_NAME + "." + distributionKey; - return new Dispatch(dispatchGroup, parent, subConfigId, nodeSpec, dispatchLevel, false, false); - } - - /** - * Override the default service-type - * @return String "topleveldispatch" - */ - public String getServiceType() { - return "topleveldispatch"; - } - - /** - * @return the startup command - */ - public String getStartupCommand() { - return "exec sbin/vespa-dispatch -c $VESPA_CONFIG_ID"; - } - - private int getFrtPort() { return getRelativePort(0); } - public int getDispatchPort() { return getRelativePort(1); } - @Override - public int getHealthPort() { return getRelativePort(2); } - - /** - * Twice the default of the number of threads in the container. - * Could have been unbounded if it was not roundrobin, but stack based usage in dispatch. - * We are not putting to much magic into this one as this will disappear as soon as - * dispatch is implemented in Java. - */ - public int getMaxThreads() { return 500*2; } - - public String getHostname() { - return getHost().getHostname(); - } - - @Override - public NodeSpec getNodeSpec() { - return nodeSpec; - } - - public String getDispatcherConnectSpec() { - return "tcp/" + getHost().getHostname() + ":" + getDispatchPort(); - } - - public DispatchGroup getDispatchGroup() { - return dispatchGroup; - } - - @Override - public void getConfig(FdispatchrcConfig.Builder builder) { - builder.ptport(getDispatchPort()). - frtport(getFrtPort()). - healthport(getHealthPort()). - maxthreads(getMaxThreads()); - if (!isTopLevel) { - builder.partition(getNodeSpec().partitionId()); - builder.dispatchlevel(dispatchLevel); - } - } - - @Override - public void getConfig(PartitionsConfig.Builder builder) { - int rowbits = dispatchGroup.getRowBits(); - final PartitionsConfig.Dataset.Builder datasetBuilder = new PartitionsConfig.Dataset.Builder(). - id(0). - searchablecopies(dispatchGroup.getSearchableCopies()). - refcost(1). - rowbits(rowbits). - numparts(dispatchGroup.getNumPartitions()). - mpp(dispatchGroup.getMinNodesPerColumn()); - if (dispatchGroup.useFixedRowInDispatch()) { - datasetBuilder.querydistribution(PartitionsConfig.Dataset.Querydistribution.Enum.FIXEDROW); - datasetBuilder.maxnodesdownperfixedrow(dispatchGroup.getMaxNodesDownPerFixedRow()); - } - if (useAdaptiveDispatch) { - datasetBuilder.useroundrobinforfixedrow(false); - } - SearchCoverage coverage = dispatchGroup.getSearchCoverage(); - if (coverage != null) { - if (coverage.getMinimum() != null) { - datasetBuilder.minimal_searchcoverage(coverage.getMinimum() * 100); // as percentage - } - if (coverage.getMinWaitAfterCoverageFactor() != null) { - datasetBuilder.higher_coverage_minsearchwait(coverage.getMinWaitAfterCoverageFactor()); - } - if (coverage.getMaxWaitAfterCoverageFactor() != null) { - datasetBuilder.higher_coverage_maxsearchwait(coverage.getMaxWaitAfterCoverageFactor()); - } - } - - Tuning tuning = dispatchGroup.getTuning(); - boolean useLocalNode = false; - if (tuning != null && tuning.dispatch != null) { - useLocalNode = tuning.dispatch.useLocalNode; - } - final List<PartitionsConfig.Dataset.Engine.Builder> allEngines = new ArrayList<>(); - for (SearchInterface searchNode : dispatchGroup.getSearchersIterable()) { - final PartitionsConfig.Dataset.Engine.Builder engineBuilder = new PartitionsConfig.Dataset.Engine.Builder(). - name_and_port(searchNode.getDispatcherConnectSpec()). - rowid(searchNode.getNodeSpec().groupIndex()). - partid(searchNode.getNodeSpec().partitionId()); - allEngines.add(engineBuilder); - if (preferLocalRow) { - if (getHostname().equals(searchNode.getHostName())) { - engineBuilder.refcost(1); - } else { - engineBuilder.refcost(Integer.MAX_VALUE); - } - } - - if (!useLocalNode || getHostname().equals(searchNode.getHostName())) { - if (useLocalNode) { - engineBuilder.rowid(0); - } - datasetBuilder.engine.add(engineBuilder); - - } - } - //Do not create empty engine list for a dataset if no local search nodes found - if(datasetBuilder.engine.isEmpty() && useLocalNode) { - for(PartitionsConfig.Dataset.Engine.Builder engineBuilder: allEngines) { - datasetBuilder.engine.add(engineBuilder); - } - } - - builder.dataset.add(datasetBuilder); - - if (tuning != null) { - tuning.getConfig(builder); - scaleMaxHitsPerPartitions(builder, tuning); - } - } - - private int getNumLeafNodesInGroup() { - int numSearchers = 0; - for (SearchInterface search : dispatchGroup.getSearchersIterable()) { - if (search instanceof Dispatch) { - numSearchers += ((Dispatch) search).getNumLeafNodesInGroup(); - } else { - numSearchers++; - } - } - if (numSearchers > 0) { - // Divide by number of partitions, otherwise we would count the same leaf node partition number of times. - return numSearchers / dispatchGroup.getNumPartitions(); - } - return 0; - } - - private void scaleMaxHitsPerPartitions(PartitionsConfig.Builder builder, Tuning tuning) { - if (tuning == null || tuning.dispatch == null || tuning.dispatch.maxHitsPerPartition == null) { - return; - } - int numLeafNodes = getNumLeafNodesInGroup(); - for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { - dataset.maxhitspernode(tuning.dispatch.maxHitsPerPartition * numLeafNodes); - } - } - - @Override - public void allocatePorts(int start, PortAllocBridge from) { - // NB: ignore "start" - from.allocatePort("rpc"); - from.allocatePort("fs4"); - from.allocatePort("health"); - } - - /** - * @return the number of ports needed - */ - public int getPortCount() { return 3; } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java index 3d57732efde..384f77737c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.vespa.model.content.SearchCoverage; - -import java.util.*; +import java.util.Map; +import java.util.TreeMap; /** * Class representing a group of @link{SearchInterface} nodes and a set of @link{Dispatch} nodes. @@ -14,8 +13,7 @@ import java.util.*; */ public class DispatchGroup { - private final List<Dispatch> dispatchers = new ArrayList<>(); - private final Map<Integer, Map<Integer, SearchInterface> > searchers = new TreeMap<>(); + private final Map<Integer, Map<Integer, SearchInterface>> searchers = new TreeMap<>(); final private IndexedSearchCluster sc; @@ -23,11 +21,6 @@ public class DispatchGroup { this.sc = sc; } - DispatchGroup addDispatcher(Dispatch dispatch) { - dispatchers.add(dispatch); - return this; - } - DispatchGroup addSearcher(SearchInterface search) { Map<Integer, SearchInterface> rows = searchers.get(search.getNodeSpec().partitionId()); if (rows == null) { @@ -43,15 +36,6 @@ public class DispatchGroup { return this; } - DispatchGroup clearSearchers() { - searchers.clear(); - return this; - } - - List<Dispatch> getDispatchers() { - return Collections.unmodifiableList(dispatchers); - } - public Iterable getSearchersIterable() { return new Iterable(searchers); } @@ -68,28 +52,12 @@ public class DispatchGroup { return sc.useFixedRowInDispatch(); } - public int getMinNodesPerColumn() { - return sc.getMinNodesPerColumn(); - } - public int getSearchableCopies() { return sc.getSearchableCopies(); } public int getMaxNodesDownPerFixedRow() { return sc.getMaxNodesDownPerFixedRow(); } - SearchCoverage getSearchCoverage() { - return sc.getSearchCoverage(); - } - - Tuning getTuning() { - return sc.getTuning(); - } - - String getClusterName() { - return sc.getClusterName(); - } - static class Iterator implements java.util.Iterator<SearchInterface> { private java.util.Iterator<Map<Integer, SearchInterface>> it1; private java.util.Iterator<SearchInterface> it2; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java index 4ff5fcab347..b489c5b9242 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.content.DispatchSpec; @@ -30,49 +28,28 @@ public class DispatchGroupBuilder { this.searchCluster = searchCluster; } - public void build(DeployLogger deployLogger, List<DispatchSpec.Group> groupsSpec, List<SearchNode> searchNodes) { + public void build(List<DispatchSpec.Group> groupsSpec, List<SearchNode> searchNodes) { Map<Integer, SearchNode> searchNodeMap = buildSearchNodeMap(searchNodes); for (int partId = 0; partId < groupsSpec.size(); ++partId) { DispatchSpec.Group groupSpec = groupsSpec.get(partId); DispatchGroup group = new DispatchGroup(searchCluster); - populateDispatchGroup(deployLogger, group, groupSpec.getNodes(), searchNodeMap, partId); + populateDispatchGroup(group, groupSpec.getNodes(), searchNodeMap, partId); } } - private void populateDispatchGroup(DeployLogger deployLogger, - DispatchGroup group, + private void populateDispatchGroup(DispatchGroup group, List<DispatchSpec.Node> nodeList, Map<Integer, SearchNode> searchNodesMap, int partId) { for (int rowId = 0; rowId < nodeList.size(); ++rowId) { int distributionKey = nodeList.get(rowId).getDistributionKey(); SearchNode searchNode = searchNodesMap.get(distributionKey); - Dispatch dispatch = buildDispatch(deployLogger, group, new NodeSpec(rowId, partId), distributionKey, searchNode.getHostResource()); - group.addDispatcher(dispatch); - rootDispatch.addSearcher(dispatch); // Note: the rowId in this context will be the partId for the underlying search node. group.addSearcher(buildSearchInterface(searchNode, rowId)); } } - /** - * Builds a mid-level dispatcher with a configId containing the same stable distribution-key as the search node it - * is located on. - * - * If this.dispatchParent has subConfigId 'dispatchers', the config ids of the mid-level - * dispatchers are '../dispatchers/dispatch.X' where X is the distribution-key of the search node. - * - * The dispatch group that will contain this mid-level dispatcher is no longer part of the config producer tree, - * but only contains information about the dispatchers and searchers in this group. - */ - private Dispatch buildDispatch(DeployLogger deployLogger, DispatchGroup group, NodeSpec nodeSpec, int distributionKey, HostResource hostResource) { - Dispatch dispatch = Dispatch.createDispatchWithStableConfigId(group, dispatchParent, nodeSpec, distributionKey, 1); - dispatch.setHostResource(hostResource); - dispatch.initService(deployLogger); - return dispatch; - } - private static SearchInterface buildSearchInterface(SearchNode searchNode, int partId) { searchNode.updatePartition(partId); // ensure that search node uses the same partId as dispatch sees return new SearchNodeWrapper(new NodeSpec(0, partId), searchNode); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 76617bf1b9f..40cc09938e2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -1,10 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.log.LogLevel; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.searchdefinition.DocumentOnlySearch; @@ -15,10 +13,6 @@ import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.SimpleConfigProducer; -import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.content.DispatchSpec; import com.yahoo.vespa.model.content.SearchCoverage; @@ -29,7 +23,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.logging.Logger; /** * @author baldersheim @@ -85,8 +78,6 @@ public class IndexedSearchCluster extends SearchCluster } } - private static final Logger log = Logger.getLogger(IndexedSearchCluster.class.getName()); - private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config. private String indexingChainName = null; @@ -103,7 +94,6 @@ public class IndexedSearchCluster extends SearchCluster private int searchableCopies = 1; - private final SimpleConfigProducer dispatchParent; private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; private final boolean useAdaptiveDispatch; @@ -122,7 +112,6 @@ public class IndexedSearchCluster extends SearchCluster public IndexedSearchCluster(AbstractConfigProducer parent, String clusterName, int index, DeployState deployState) { super(parent, clusterName, index); unionCfg = new UnionConfiguration(this, documentDbs); - dispatchParent = new SimpleConfigProducer(this, "dispatchers"); rootDispatch = new DispatchGroup(this); useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch(); } @@ -183,46 +172,6 @@ public class IndexedSearchCluster extends SearchCluster return this; } - public Dispatch addTld(DeployLogger deployLogger, AbstractConfigProducer tldParent, HostResource hostResource) { - int index = rootDispatch.getDispatchers().size(); - Dispatch tld = Dispatch.createTld(rootDispatch, tldParent, index); - tld.useAdaptiveDispatch(useAdaptiveDispatch); - tld.setHostResource(hostResource); - tld.initService(deployLogger); - rootDispatch.addDispatcher(tld); - return tld; - } - - /** - * Make sure to allocate tld with same id as container (i.e if container cluster name is 'foo', with containers - * with index 0,1,2 the tlds created will get names ../foo.0.tld.0, ../foo.1.tld.1, ../foo.2.tld.2, so that tld config id is - * stable no matter what changes are done to the number of containers in a container cluster - * @param tldParent the indexed search cluster the tlds to add should be connected to - * @param containerCluster the container cluster that should use the tlds created for searching the indexed search cluster above - */ - public void addTldsWithSameIdsAsContainers(DeployLogger deployLogger, AbstractConfigProducer tldParent, ContainerCluster<? extends Container> containerCluster) { - for (Container container : containerCluster.getContainers()) { - String containerSubId = container.getSubId(); - if ( ! containerSubId.contains(".")) { - throw new RuntimeException("Expected container sub id to be of the form string.number"); - } - int containerIndex = Integer.parseInt(containerSubId.split("\\.")[1]); - String containerClusterName = containerCluster.getName(); - log.log(LogLevel.DEBUG, "Adding tld with index " + containerIndex + " for content cluster " + this.getClusterName() + - ", container cluster " + containerClusterName + " (container id " + containerSubId + - ") on host " + container.getHostResource().getHostname()); - rootDispatch.addDispatcher(createTld(deployLogger, tldParent, container.getHostResource(), containerClusterName, containerIndex).useAdaptiveDispatch(useAdaptiveDispatch)); - } - } - - private Dispatch createTld(DeployLogger deployLogger, AbstractConfigProducer tldParent, HostResource hostResource, String containerClusterName, int containerIndex) { - Dispatch tld = Dispatch.createTldWithContainerIdInName(rootDispatch, tldParent, containerClusterName, containerIndex); - tld.useAdaptiveDispatch(useAdaptiveDispatch); - tld.setHostResource(hostResource); - tld.initService(deployLogger); - return tld; - } - public DispatchGroup getRootDispatch() { return rootDispatch; } public void addSearcher(SearchNode searcher) { @@ -230,8 +179,6 @@ public class IndexedSearchCluster extends SearchCluster rootDispatch.addSearcher(searcher); } - public List<Dispatch> getTLDs() { return rootDispatch.getDispatchers(); } - public List<SearchNode> getSearchNodes() { return Collections.unmodifiableList(searchNodes); } public int getSearchNodeCount() { return searchNodes.size(); } public SearchNode getSearchNode(int index) { return searchNodes.get(index); } @@ -327,10 +274,6 @@ public class IndexedSearchCluster extends SearchCluster this.searchCoverage = searchCoverage; } - SearchCoverage getSearchCoverage() { - return searchCoverage; - } - @Override public DerivedConfiguration getSdConfig() { return null; } @@ -357,8 +300,6 @@ public class IndexedSearchCluster extends SearchCluster @Override protected void exportSdFiles(File toDir) { } - int getMinNodesPerColumn() { return 0; } - boolean useFixedRowInDispatch() { for (SearchNode node : getSearchNodes()) { if (node.getNodeSpec().groupIndex() > 0) { @@ -397,18 +338,6 @@ public class IndexedSearchCluster extends SearchCluster return dispatchSpec; } - public boolean useMultilevelDispatchSetup() { - return dispatchSpec != null && dispatchSpec.getGroups() != null && !dispatchSpec.getGroups().isEmpty(); - } - - public void setupDispatchGroups(DeployLogger deployLogger) { - if (!useMultilevelDispatchSetup()) { - return; - } - rootDispatch.clearSearchers(); - new DispatchGroupBuilder(dispatchParent, rootDispatch, this).build(deployLogger, dispatchSpec.getGroups(), getSearchNodes()); - } - @Override public void getConfig(DispatchConfig.Builder builder) { for (SearchNode node : getSearchNodes()) { @@ -420,6 +349,9 @@ public class IndexedSearchCluster extends SearchCluster nodeBuilder.fs4port(node.getDispatchPort()); builder.node(nodeBuilder); } + if (useAdaptiveDispatch) + builder.distributionPolicy(DistributionPolicy.ADAPTIVE); + if (tuning.dispatch.minActiveDocsCoverage != null) builder.minActivedocsPercentage(tuning.dispatch.minActiveDocsCoverage); if (tuning.dispatch.minGroupCoverage != null) @@ -434,8 +366,10 @@ public class IndexedSearchCluster extends SearchCluster break; } } + if (tuning.dispatch.maxHitsPerPartition != null) + builder.maxHitsPerNode(tuning.dispatch.maxHitsPerPartition); + builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); - builder.useMultilevelDispatch(useMultilevelDispatchSetup()); builder.useLocalNode(tuning.dispatch.useLocalNode); builder.searchableCopies(rootDispatch.getSearchableCopies()); if (searchCoverage != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index b0fe2877386..fac641bd714 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.config.search.core.PartitionsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.TuningDispatch; @@ -14,47 +13,15 @@ import static com.yahoo.text.Lowercase.toLowerCase; * * @author geirst */ -public class Tuning extends AbstractConfigProducer implements PartitionsConfig.Producer, ProtonConfig.Producer { +public class Tuning extends AbstractConfigProducer implements ProtonConfig.Producer { - public static class Dispatch implements PartitionsConfig.Producer { + public static class Dispatch { public Integer maxHitsPerPartition = null; public TuningDispatch.DispatchPolicy policy = null; public boolean useLocalNode = false; public Double minGroupCoverage = null; public Double minActiveDocsCoverage = null; - - @Override - public void getConfig(PartitionsConfig.Builder builder) { - if (maxHitsPerPartition != null) { - for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { - dataset.maxhitspernode(maxHitsPerPartition); - } - } - if (minGroupCoverage != null) { - for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { - dataset.min_group_coverage(minGroupCoverage); - } - } - if (minActiveDocsCoverage != null) { - for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { - dataset.min_activedocs_coverage(minActiveDocsCoverage); - } - } - if (policy != null) { - for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { - switch (policy) { - case ADAPTIVE: - dataset.useroundrobinforfixedrow(false); - break; - case ROUNDROBIN: - default: - dataset.useroundrobinforfixedrow(true); - break; - } - } - } - } } public static class SearchNode implements ProtonConfig.Producer { @@ -432,13 +399,6 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P } @Override - public void getConfig(PartitionsConfig.Builder builder) { - if (dispatch != null) { - dispatch.getConfig(builder); - } - } - - @Override public void getConfig(ProtonConfig.Builder builder) { if (searchNode != null) searchNode.getConfig(builder); } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 535048eafa3..dfd2fe00d46 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -7,13 +7,10 @@ import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.Zone; -import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; -import com.yahoo.vespa.config.search.core.PartitionsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; @@ -22,14 +19,12 @@ import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Logserver; import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; -import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.StorageNode; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.search.Dispatch; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; @@ -38,7 +33,6 @@ import org.junit.Ignore; import org.junit.Test; import java.io.StringReader; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -1019,10 +1013,6 @@ public class ModelProvisioningTest { assertThat(cluster.getRootGroup().getNodes().get(2).getConfigId(), is("bar/storage/2")); assertThat(cluster.getRootGroup().getNodes().get(3).getDistributionKey(), is(3)); assertThat(cluster.getRootGroup().getNodes().get(3).getConfigId(), is("bar/storage/3")); - PartitionsConfig.Builder partBuilder = new PartitionsConfig.Builder(); - cluster.getSearch().getIndexed().getTLDs().get(0).getConfig(partBuilder); - PartitionsConfig partCFg = partBuilder.build(); - assertEquals(4, partCFg.dataset(0).searchablecopies()); } @Test @@ -1605,97 +1595,6 @@ public class ModelProvisioningTest { return modelCreatorWithMockPkg.create(false, deployState); } - @Test - public void testThatTldConfigIdsAreDeterministic() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>" + - " <admin version='4.0'/>" + - " <container version='1.0' id='jdisc0'>" + - " <search/>" + - " <nodes count='2'/>" + - " </container>" + - " <container version='1.0' id='jdisc1'>" + - " <search/>" + - " <nodes count='2'/>" + - " </container>" + - " <content version='1.0' id='content0'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='2'/>" + - " </content>" + - " <content version='1.0' id='content1'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='2'/>" + - " </content>" + - "</services>"; - - int numberOfHosts = 8; - - { - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); - // Nodes used will be default0, default1, .. and so on. - VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - - Map<String, ContentCluster> contentClusters = model.getContentClusters(); - assertEquals(2, contentClusters.size()); - - checkThatTldAndContainerRunningOnSameHostHaveSameId( - model.getContainerClusters().values(), - model.getContentClusters().values(), - 0); - } - - { - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(numberOfHosts + 1); - // Start numbering nodes with index 1 and retire first node - // Nodes used will be default1, default2, .. and so on. Containers will start with index 1, not 0 as they are in the test above - VespaModel model = tester.createModel(services, true, 1, "default0"); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - - Map<String, ContentCluster> contentClusters = model.getContentClusters(); - assertEquals(2, contentClusters.size()); - - checkThatTldAndContainerRunningOnSameHostHaveSameId( - model.getContainerClusters().values(), - model.getContentClusters().values(), - 1); - } - } - - private void checkThatTldAndContainerRunningOnSameHostHaveSameId(Collection<ApplicationContainerCluster> containerClusters, - Collection<ContentCluster> contentClusters, - int startIndexForContainerIds) { - for (ContentCluster contentCluster : contentClusters) { - String contentClusterName = contentCluster.getName(); - int i = 0; - for (ApplicationContainerCluster containerCluster : containerClusters) { - String containerClusterName = containerCluster.getName(); - for (int j = 0; j < 2; j++) { - Dispatch tld = contentCluster.getSearch().getIndexed().getTLDs().get(2 * i + j); - ApplicationContainer container = containerCluster.getContainers().get(j); - int containerConfigIdIndex = j + startIndexForContainerIds; - - assertEquals(container.getHostName(), tld.getHostname()); - assertEquals(contentClusterName + "/search/cluster." + contentClusterName + "/tlds/" + - containerClusterName + "." + containerConfigIdIndex + ".tld." + containerConfigIdIndex, - tld.getConfigId()); - assertEquals(containerClusterName + "/" + "container." + containerConfigIdIndex, - container.getConfigId()); - } - i++; - } - } - } - private int physicalMemoryPercentage(ContainerCluster cluster) { QrStartConfig.Builder b = new QrStartConfig.Builder(); cluster.getConfig(b); @@ -1725,11 +1624,6 @@ public class ModelProvisioningTest { assertEquals(40, getProtonConfig(cluster, 1).hwinfo().disk().writespeed(), 0.001); } - private static Flavor createFlavorFromDiskSetting(String name, boolean fastDisk) { - return new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder(). - name(name).fastDisk(fastDisk))); - } - private static ProtonConfig getProtonConfig(ContentSearchCluster cluster, int searchNodeIdx) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); List<SearchNode> searchNodes = cluster.getSearchNodes(); @@ -1787,11 +1681,6 @@ public class ModelProvisioningTest { private static long GB = 1024 * 1024 * 1024; - private static Flavor createFlavorFromMemoryAndDisk(String name, int memoryGb, int diskGb) { - return new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder(). - name(name).minMainMemoryAvailableGb(memoryGb).minDiskAvailableGb(diskGb))); - } - private static ProtonConfig getProtonConfig(VespaModel model, String configId) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); model.getConfig(builder, configId); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 567e54ed4c4..a7ab9e02a79 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -12,7 +12,6 @@ import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.config.GenericConfig; -import com.yahoo.vespa.config.search.core.PartitionsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.Service; @@ -21,7 +20,6 @@ import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.Dispatch; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.search.StreamingSearchCluster; @@ -172,25 +170,11 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. assertEquals(1, c.getRoot().getHostSystem().getHosts().size()); HostResource h = c.getRoot().getHostSystem().getHost("mockhost"); - String [] expectedServices = {"logd", "configproxy","config-sentinel", "qrserver", "storagenode", "searchnode", "distributor", "topleveldispatch", "transactionlogserver"}; -// TODO assertServices(h, expectedServices); + String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy","config-sentinel", "qrserver", "storagenode", "searchnode", "distributor", "transactionlogserver"}; + assertServices(h, expectedServices); assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); assertEquals("clu/search/cluster.clu/0", h.getService("searchnode").getConfigId()); assertEquals("clu/distributor/0", h.getService("distributor").getConfigId()); - assertEquals("clu/search/cluster.clu/tlds/qrc.0.tld.0", h.getService("topleveldispatch").getConfigId()); - //assertEquals("tcp/node0:19104", h.getService("topleveldispatch").getConfig("partitions", "").innerArray("dataset").value("0").innerArray("engine").value("0").getString("name_and_port")); - PartitionsConfig partitionsConfig = new PartitionsConfig((PartitionsConfig.Builder) - m.getConfig(new PartitionsConfig.Builder(), "clu/search/cluster.clu/tlds/qrc.0.tld.0")); - assertTrue(partitionsConfig.dataset(0).engine(0).name_and_port().startsWith("tcp/node0:191")); - } - - @Test - public void testConfigIdLookup() { - VespaModel m = new VespaModelCreatorWithMockPkg(createAppWithMusic(getHosts(), getBasicServices())).create(); - - PartitionsConfig partitionsConfig = new PartitionsConfig((PartitionsConfig.Builder) - m.getConfig(new PartitionsConfig.Builder(), "clu/search/cluster.clu/tlds/qrc.0.tld.0")); - assertTrue(partitionsConfig.dataset(0).engine(0).name_and_port().startsWith("tcp/node0:191")); } @Test @@ -198,9 +182,6 @@ public class ContentBuilderTest extends DomBuilderTest { String services = getServices("<node hostalias='mockhost' distribution-key='0'/>" + "<node hostalias='mockhost' distribution-key='1'/>"); VespaModel m = new VespaModelCreatorWithMockPkg(createAppWithMusic(getHosts(), services)).create(); - PartitionsConfig partitionsConfig = new PartitionsConfig((PartitionsConfig.Builder) - m.getConfig(new PartitionsConfig.Builder(), "clu/search/cluster.clu/tlds/qrc.0.tld.0")); - assertTrue(partitionsConfig.dataset(0).engine(0).name_and_port().startsWith("tcp/node0:191")); IndexedSearchCluster sc = m.getContentClusters().get("clu").getSearch().getIndexed(); assertEquals(2, sc.getSearchNodeCount()); } @@ -711,45 +692,6 @@ public class ContentBuilderTest extends DomBuilderTest { } @Test - public void requireOneTldPerSearchContainer() { - ContentCluster content = createContent( - " <content version='1.0' id='storage'>\n" + - " <redundancy>1</redundancy>\n" + - " <documents>" + - " <document type='music' mode='index'/>" + - " </documents>" + - " <group>\n" + - " <node hostalias='mockhost' distribution-key='0' />\n" + - " </group>\n" + - " </content>\n" + - " <container version='1.0' id='qrc'>" + - " <search/>" + - " <nodes>" + - " <node hostalias='mockhost' />" + - " </nodes>" + - " </container>" + - " <container version='1.0' id='qrc2'>" + - " <http>" + - " <server id ='server1' port='5000' />" + - " </http>" + - " <search/>" + - " <nodes>" + - " <node hostalias='mockhost' />" + - " <node hostalias='mockhost2' />" + - " </nodes>" + - " </container>" - - ); - List<Dispatch> tlds = content.getSearch().getIndexed().getTLDs(); - - assertThat(tlds.get(0).getHostname(), is("node0")); - assertThat(tlds.get(1).getHostname(), is("node0")); - assertThat(tlds.get(2).getHostname(), is("node1")); - - assertThat(tlds.size(), is(3)); - } - - @Test @Ignore public void ensureOverrideAppendedOnlyOnce() { ContentCluster content = createContent( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index d98d1da9d2a..d6579ba88d4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; -import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; @@ -271,25 +271,6 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(1, cluster.getContainers().size()); } - private void verifyRoundRobinPropertiesControl(boolean useAdaptiveDispatch) { - VespaModel model = createEnd2EndOneNode(new TestProperties().setUseAdaptiveDispatch(useAdaptiveDispatch)); - - ContentCluster cc = model.getContentClusters().get("storage"); - PartitionsConfig.Builder partBuilder = new PartitionsConfig.Builder(); - assertNotNull(cc.getSearch()); - assertNotNull(cc.getSearch().getIndexed()); - assertEquals(1, cc.getSearch().getIndexed().getTLDs().size()); - cc.getSearch().getIndexed().getTLDs().get(0).getConfig(partBuilder); - PartitionsConfig partitionsConfig = new PartitionsConfig(partBuilder); - assertFalse(useAdaptiveDispatch == partitionsConfig.dataset(0).useroundrobinforfixedrow()); - } - - @Test - public void default_dispatch_controlled_by_properties() { - verifyRoundRobinPropertiesControl(false); - verifyRoundRobinPropertiesControl(true); - } - @Test public void testSearchTuning() { String xml = @@ -950,4 +931,27 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(distributionBits, storDistributormanagerConfig.minsplitcount()); } + private void verifyRoundRobinPropertiesControl(boolean useAdaptiveDispatch) { + VespaModel model = createEnd2EndOneNode(new TestProperties().setUseAdaptiveDispatch(useAdaptiveDispatch)); + + ContentCluster cc = model.getContentClusters().get("storage"); + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + cc.getSearch().getConfig(builder); + + DispatchConfig cfg = new DispatchConfig(builder); + if (useAdaptiveDispatch) { + assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, cfg.distributionPolicy()); + } else { + assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, cfg.distributionPolicy()); + } + + } + + @Test + public void default_dispatch_controlled_by_properties() { + verifyRoundRobinPropertiesControl(false); + verifyRoundRobinPropertiesControl(true); + } + + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java index afedbaea779..d290d4ec953 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.StorDistributionConfig; -import com.yahoo.vespa.config.search.core.PartitionsConfig; import com.yahoo.config.model.test.MockRoot; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; @@ -21,8 +20,6 @@ import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.hamcrest.Matchers.containsString; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClusterXml; -import static com.yahoo.vespa.model.search.utils.DispatchUtils.assertEngine; -import static com.yahoo.vespa.model.search.utils.DispatchUtils.getDataset; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -35,19 +32,13 @@ import static org.junit.Assert.assertTrue; */ public class IndexedHierarchicDistributionTest { - private ContentCluster addDispatcher(ContentCluster c) { - MockRoot root = new MockRoot(""); - c.getSearch().getIndexed().addTld(root.deployLogger(), new SimpleConfigProducer(root, ""), new HostResource(new Host(root, "mockhost"))); - return c; - } - private ContentCluster getOneGroupCluster() throws Exception { String groupXml = joinLines(" <group>", " <node distribution-key='0' hostalias='mockhost'/>", " <node distribution-key='1' hostalias='mockhost'/>", " <node distribution-key='2' hostalias='mockhost'/>", " </group>", ""); - return addDispatcher(createCluster(createClusterXml(groupXml, 2, 2))); + return createCluster(createClusterXml(groupXml, 2, 2)); } private String getTwoGroupsXml(String partitions) { @@ -67,11 +58,11 @@ public class IndexedHierarchicDistributionTest { } private ContentCluster getTwoGroupsCluster() throws Exception { - return addDispatcher(createCluster(createClusterXml(getTwoGroupsXml("3|*"), 6, 6))); + return createCluster(createClusterXml(getTwoGroupsXml("3|*"), 6, 6)); } private ContentCluster getTwoGroupsCluster(int redundancy, int searchableCopies, String partitions) throws Exception { - return addDispatcher(createCluster(createClusterXml(getTwoGroupsXml(partitions), redundancy, searchableCopies))); + return createCluster(createClusterXml(getTwoGroupsXml(partitions), redundancy, searchableCopies)); } private void assertSearchNode(int expRowId, int expPartitionId, int expDistibutionKey, SearchNode node) { @@ -98,20 +89,6 @@ public class IndexedHierarchicDistributionTest { } @Test - public void requireThatDispatcherIsCorrectWithOneGroup() throws Exception { - ContentCluster c = getOneGroupCluster(); - PartitionsConfig.Dataset dataset = getDataset(c.getSearch().getIndexed().getTLDs().get(0)); - - assertEquals(3, dataset.numparts()); - assertEquals(PartitionsConfig.Dataset.Querydistribution.AUTOMATIC, dataset.querydistribution()); - List<PartitionsConfig.Dataset.Engine> engines = dataset.engine(); - assertEquals(3, engines.size()); - assertEngine(0, 0, engines.get(0)); - assertEngine(0, 1, engines.get(1)); - assertEngine(0, 2, engines.get(2)); - } - - @Test public void requireThatActivePerLeafGroupIsDefaultWithOneGroup() throws Exception { ContentCluster c = getOneGroupCluster(); assertFalse(getStorDistributionConfig(c).active_per_leaf_group()); @@ -132,24 +109,6 @@ public class IndexedHierarchicDistributionTest { } @Test - public void requireThatDispatcherIsCorrectWithTwoGroups() throws Exception { - ContentCluster c = getTwoGroupsCluster(); - PartitionsConfig.Dataset dataset = getDataset(c.getSearch().getIndexed().getTLDs().get(0)); - - assertEquals(3, dataset.numparts()); - assertEquals(2, dataset.maxnodesdownperfixedrow()); - assertEquals(PartitionsConfig.Dataset.Querydistribution.FIXEDROW, dataset.querydistribution()); - List<PartitionsConfig.Dataset.Engine> engines = dataset.engine(); - assertEquals(6, engines.size()); - assertEngine(0, 0, engines.get(0)); - assertEngine(1, 0, engines.get(1)); - assertEngine(0, 1, engines.get(2)); - assertEngine(1, 1, engines.get(3)); - assertEngine(0, 2, engines.get(4)); - assertEngine(1, 2, engines.get(5)); - } - - @Test public void requireThatActivePerLeafGroupIsSetWithTwoGroups() throws Exception { ContentCluster c = getTwoGroupsCluster(); assertTrue(getStorDistributionConfig(c).active_per_leaf_group()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 4fadea74feb..4291c68eff8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -4,10 +4,9 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; -import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.Content; -import com.yahoo.vespa.model.search.Dispatch; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import org.junit.Test; @@ -16,6 +15,7 @@ import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; */ public class ClusterTest { + static final double DELTA = 1E-12; @Test public void requireThatContentSearchIsApplied() { ContentCluster cluster = newContentCluster(joinLines("<search>", @@ -32,10 +33,19 @@ public class ClusterTest { "</search>")); IndexedSearchCluster searchCluster = cluster.getSearch().getIndexed(); assertNotNull(searchCluster); - assertEquals(1.1, searchCluster.getQueryTimeout(), 1E-6); - assertEquals(2.3, searchCluster.getVisibilityDelay(), 1E-6); + assertEquals(1.1, searchCluster.getQueryTimeout(), DELTA); + assertEquals(2.3, searchCluster.getVisibilityDelay(), DELTA); ProtonConfig proton = getProtonConfig(cluster); - assertEquals(searchCluster.getVisibilityDelay(), proton.documentdb(0).visibilitydelay(), 1E-6); + assertEquals(searchCluster.getVisibilityDelay(), proton.documentdb(0).visibilitydelay(), DELTA); + } + + @Test + public void requireThatVisibilityDelayIsZeroForGlobalDocumentType() { + ContentCluster cluster = newContentCluster(joinLines("<search>", + " <visibility-delay>2.3</visibility-delay>", + "</search>"), true); + ProtonConfig proton = getProtonConfig(cluster); + assertEquals(0.0, proton.documentdb(0).visibilitydelay(), DELTA); } @Test @@ -47,56 +57,93 @@ public class ClusterTest { " <max-wait-after-coverage-factor>0.58</max-wait-after-coverage-factor>", " </coverage>", "</search>")); - assertEquals(1, cluster.getSearch().getIndexed().getTLDs().size()); - for (Dispatch tld : cluster.getSearch().getIndexed().getTLDs()) { - PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); - tld.getConfig(builder); - PartitionsConfig config = new PartitionsConfig(builder); - assertEquals(11.0, config.dataset(0).minimal_searchcoverage(), 1E-6); - assertEquals(0.23, config.dataset(0).higher_coverage_minsearchwait(), 1E-6); - assertEquals(0.58, config.dataset(0).higher_coverage_maxsearchwait(), 1E-6); - assertEquals(2, config.dataset(0).searchablecopies()); - assertTrue(config.dataset(0).useroundrobinforfixedrow()); - } + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + cluster.getSearch().getConfig(builder); + DispatchConfig config = new DispatchConfig(builder); + assertEquals(11.0, config.minSearchCoverage(), DELTA); + assertEquals(0.23, config.minWaitAfterCoverageFactor(), DELTA); + assertEquals(0.58, config.maxWaitAfterCoverageFactor(), DELTA); + assertEquals(2, config.searchableCopies()); + assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); } @Test - public void requireThatDispatchTuningIsApplied() { + public void requireThatDispatchTuningIsApplied() { ContentCluster cluster = newContentCluster(joinLines("<search>", "</search>"), - joinLines("<tuning>", - "</tuning>")); - assertEquals(1, cluster.getSearch().getIndexed().getTLDs().size()); - for (Dispatch tld : cluster.getSearch().getIndexed().getTLDs()) { - PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); - tld.getConfig(builder); - PartitionsConfig config = new PartitionsConfig(builder); - assertEquals(2, config.dataset(0).searchablecopies()); - assertTrue(config.dataset(0).useroundrobinforfixedrow()); - } + "", + joinLines( + "<max-hits-per-partition>77</max-hits-per-partition>", + "<dispatch-policy>adaptive</dispatch-policy>", + "<min-group-coverage>13</min-group-coverage>", + "<min-active-docs-coverage>93</min-active-docs-coverage>", + "<use-local-node>true</use-local-node>"), + false); + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + cluster.getSearch().getConfig(builder); + DispatchConfig config = new DispatchConfig(builder); + assertEquals(2, config.searchableCopies()); + assertEquals(93.0, config.minActivedocsPercentage(), DELTA); + assertEquals(13.0, config.minGroupCoverage(), DELTA); + assertTrue(config.useLocalNode()); + assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); + assertEquals(77, config.maxHitsPerNode()); } @Test - public void requireThatVisibilityDelayIsZeroForGlobalDocumentType() { - ContentCluster cluster = newContentCluster(joinLines("<search>", - " <visibility-delay>2.3</visibility-delay>", - "</search>"), true); - ProtonConfig proton = getProtonConfig(cluster); - assertEquals(0.0, proton.documentdb(0).visibilitydelay(), 1E-6); - } + public void requireThatDefaultDispatchConfigIsCorrect() { + ContentCluster cluster = newContentCluster(joinLines("<search>", "</search>"), + joinLines("<tuning>", "</tuning>")); + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + cluster.getSearch().getConfig(builder); + DispatchConfig config = new DispatchConfig(builder); + assertEquals(2, config.searchableCopies()); + assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); + assertEquals(0, config.maxNodesDownPerGroup()); + assertEquals(1.0, config.maxWaitAfterCoverageFactor(), DELTA); + assertEquals(0, config.minWaitAfterCoverageFactor(), DELTA); + assertEquals(8, config.numJrtConnectionsPerNode()); + assertEquals(8, config.numJrtTransportThreads()); + assertEquals(100.0, config.minSearchCoverage(), DELTA); + assertEquals(97.0, config.minActivedocsPercentage(), DELTA); + assertEquals(100.0, config.minGroupCoverage(), DELTA); + assertFalse(config.useLocalNode()); + assertEquals(3, config.node().size()); + assertEquals(0, config.node(0).key()); + assertEquals(1, config.node(1).key()); + assertEquals(2, config.node(2).key()); - private static ContentCluster newContentCluster(String contentSearchXml) { - return newContentCluster(contentSearchXml, "", false); + assertEquals(19106, config.node(0).port()); + assertEquals(19118, config.node(1).port()); + assertEquals(19130, config.node(2).port()); + + assertEquals(19107, config.node(0).fs4port()); + assertEquals(19119, config.node(1).fs4port()); + assertEquals(19131, config.node(2).fs4port()); + + assertEquals(0, config.node(0).group()); + assertEquals(0, config.node(1).group()); + assertEquals(0, config.node(2).group()); + + assertEquals("localhost", config.node(0).host()); + assertEquals("localhost", config.node(1).host()); + assertEquals("localhost", config.node(2).host()); } private static ContentCluster newContentCluster(String contentSearchXml, String searchNodeTuningXml) { - return newContentCluster(contentSearchXml, searchNodeTuningXml, false); + return newContentCluster(contentSearchXml, searchNodeTuningXml, "", false); + } + + private static ContentCluster newContentCluster(String contentSearchXml) { + return newContentCluster(contentSearchXml, false); } private static ContentCluster newContentCluster(String contentSearchXml, boolean globalDocType) { - return newContentCluster(contentSearchXml, "", globalDocType); + return newContentCluster(contentSearchXml, "", "", globalDocType); } - private static ContentCluster newContentCluster(String contentSearchXml, String searchNodeTuningXml, boolean globalDocType) { + private static ContentCluster newContentCluster(String contentSearchXml, String searchNodeTuningXml, + String dispatchTuning, boolean globalDocType) + { ApplicationPackage app = new MockApplicationPackage.Builder() .withHosts(joinLines( "<hosts>", @@ -128,6 +175,11 @@ public class ClusterTest { " <node hostalias='my_host' distribution-key='2' />", " </group>", contentSearchXml, + " <tuning>", + " <dispatch>", + dispatchTuning, + " </dispatch>", + " </tuning>", " </content>", "</services>")) .withSearchDefinitions(ApplicationPackageUtils.generateSearchDefinition("my_document")) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/MultilevelDispatchTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/MultilevelDispatchTest.java deleted file mode 100644 index c87774cf692..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/MultilevelDispatchTest.java +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.model.test.MockRoot; -import com.yahoo.config.model.test.TestDriver; -import com.yahoo.config.model.test.TestRoot; -import com.yahoo.vespa.config.search.core.PartitionsConfig; -import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.SimpleConfigProducer; -import com.yahoo.vespa.model.content.DispatchSpec; -import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.content.utils.ContentClusterUtils; -import com.yahoo.vespa.model.search.utils.DispatchUtils; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClusterXml; -import static com.yahoo.vespa.model.search.utils.DispatchUtils.getDataset; -import static com.yahoo.vespa.model.search.utils.DispatchUtils.getFdispatchrcConfig; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.hamcrest.Matchers.containsString; - -/** - * Unit tests for multi-level dispatchers in an indexed content cluster. - * - * @author geirst - */ -public class MultilevelDispatchTest { - - private static class EngineAsserter { - private List<PartitionsConfig.Dataset.Engine> engines; - private int engineIdx = 0; - public EngineAsserter(int numParts, int numEngines, Dispatch dispatch) { - PartitionsConfig.Dataset dataset = getDataset(dispatch); - assertEquals(numParts, dataset.numparts()); - assertEquals(PartitionsConfig.Dataset.Querydistribution.AUTOMATIC, dataset.querydistribution()); - engines = dataset.engine(); - assertEquals(numEngines, engines.size()); - } - EngineAsserter assertEngine(int rowId, int partitionId, String connectSpec) { - DispatchUtils.assertEngine(rowId, partitionId, connectSpec, engines.get(engineIdx++)); - return this; - } - } - - private String getGroupXml() { - return " <group>\n" + - " <node distribution-key='10' hostalias='mh0'/>\n" + - " <node distribution-key='11' hostalias='mh1'/>\n" + - " <node distribution-key='12' hostalias='mh2'/>\n" + - " <node distribution-key='13' hostalias='mh3'/>\n" + - " <node distribution-key='14' hostalias='mh4'/>\n" + - " <node distribution-key='15' hostalias='mh5'/>\n" + - " </group>\n"; - } - - private String getSimpleDispatchXml() { - return " <dispatch>\n" + - " <num-dispatch-groups>2</num-dispatch-groups>\n" + - " </dispatch>\n"; - } - - private String getDispatchXml() { - return " <dispatch>\n" + - " <group>\n" + - " <node distribution-key='10'/>\n" + - " <node distribution-key='12'/>\n" + - " <node distribution-key='14'/>\n" + - " </group>\n" + - " <group>\n" + - " <node distribution-key='11'/>\n" + - " <node distribution-key='13'/>\n" + - " <node distribution-key='15'/>\n" + - " </group>\n" + - " </dispatch>\n"; - } - - private ContentCluster createCluster(String dispatchXml) throws Exception { - String[] hosts = {"mh0", "mh1", "mh2", "mh3", "mh4", "mh5"}; - MockRoot root = ContentClusterUtils.createMockRoot(hosts); - ContentCluster cluster = ContentClusterUtils.createCluster(createClusterXml(getGroupXml(), Optional.of(dispatchXml), 1, 1), root); - - AbstractConfigProducer<Dispatch> dispatchParent = new SimpleConfigProducer<>(root, "tlds"); - HostResource hostResource = new HostResource(new Host(root, "mockhost")); - IndexedSearchCluster index = cluster.getSearch().getIndexed(); - index.addTld(root.deployLogger(), dispatchParent, hostResource); - index.setupDispatchGroups(root.deployLogger()); - - root.freezeModelTopology(); - cluster.validate(); - return cluster; - } - - private List<Dispatch> getDispatchers(Dispatch tld) { - DispatchGroup group = tld.getDispatchGroup(); - List<Dispatch> dispatchers = new ArrayList<>(); - for (SearchInterface dispatch : group.getSearchersIterable()) { - dispatchers.add((Dispatch)dispatch); - } - return dispatchers; - } - - private void assertDispatchAndSearchNodes(int partId, Dispatch[] dispatchers, String[] connectSpecs, SearchNode[] searchNodes) { - assertEquals(dispatchers.length, connectSpecs.length); - assertEquals(connectSpecs.length, searchNodes.length); - int searchNodeIdx = 0; - for (int rowId = 0; rowId < dispatchers.length; ++rowId) { - assertDispatchAndSearchNodes(rowId, partId, searchNodes[searchNodeIdx++].getDistributionKey(), - dispatchers[rowId], connectSpecs, searchNodes); - } - } - - private void assertDispatchAndSearchNodes(int expRowId, int expPartId, int expDistributionKey, Dispatch dispatch, String[] connectSpecs, SearchNode[] searchNodes) { - assertEquals(expRowId, dispatch.getNodeSpec().groupIndex()); - assertEquals(expPartId, dispatch.getNodeSpec().partitionId()); - assertEquals("mycluster/search/cluster.mycluster/dispatchers/dispatch." + expDistributionKey, dispatch.getConfigId()); - assertEquals(expPartId, getFdispatchrcConfig(dispatch).partition()); - assertEquals(1, getFdispatchrcConfig(dispatch).dispatchlevel()); - - int numEngines = connectSpecs.length; - EngineAsserter ea = new EngineAsserter(numEngines, numEngines, dispatch); - for (int i = 0; i < numEngines; ++i) { - ea.assertEngine(0, i, connectSpecs[i]); - assertEquals(i, searchNodes[i].getNodeSpec().partitionId()); - } - } - - @Test - public void requireThatDispatchGroupsCanBeAutomaticallySetup() throws Exception { - ContentCluster cr = createCluster(getSimpleDispatchXml()); - IndexedSearchCluster ix = cr.getSearch().getIndexed(); - Dispatch tld = cr.getSearch().getIndexed().getTLDs().get(0); - - assertEquals("tlds/tld.0", tld.getConfigId()); - assertEquals(0, getFdispatchrcConfig(tld).dispatchlevel()); - new EngineAsserter(2, 6, tld). - assertEngine(0, 0, "tcp/mh0:19113"). - assertEngine(1, 0, "tcp/mh1:19113"). - assertEngine(2, 0, "tcp/mh2:19113"). - assertEngine(0, 1, "tcp/mh3:19113"). - assertEngine(1, 1, "tcp/mh4:19113"). - assertEngine(2, 1, "tcp/mh5:19113"); - - List<Dispatch> ds = getDispatchers(tld); - assertEquals(6, ds.size()); - { // dispatch group 1 - Dispatch[] dispatchers = {ds.get(0), ds.get(1), ds.get(2)}; - String[] specs = {"tcp/mh0:19104", "tcp/mh1:19104", "tcp/mh2:19104"}; - SearchNode[] searchNodes = {ix.getSearchNode(0), ix.getSearchNode(1), ix.getSearchNode(2)}; - assertDispatchAndSearchNodes(0, dispatchers, specs, searchNodes); - } - { // dispatch group 2 - Dispatch[] dispatchers = {ds.get(3), ds.get(4), ds.get(5)}; - String[] specs = {"tcp/mh3:19104", "tcp/mh4:19104", "tcp/mh5:19104"}; - SearchNode[] searchNodes = {ix.getSearchNode(3), ix.getSearchNode(4), ix.getSearchNode(5)}; - assertDispatchAndSearchNodes(1, dispatchers, specs, searchNodes); - } - } - - @Test - public void requireThatMaxHitsIsScaled() throws Exception { - ContentCluster cr = createCluster(getSimpleDispatchXml() + getMaxhitsTuning()); - Dispatch tld = cr.getSearch().getIndexed().getTLDs().get(0); - PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); - tld.getConfig(builder); - PartitionsConfig config = new PartitionsConfig(builder); - assertThat(config.dataset().size(), is(1)); - assertThat(config.dataset(0).maxhitspernode(), is(300)); - for (Dispatch dispatch : getDispatchers(tld)) { - PartitionsConfig.Builder b = new PartitionsConfig.Builder(); - dispatch.getConfig(b); - PartitionsConfig c= new PartitionsConfig(b); - assertThat(c.dataset().size(), is(1)); - assertThat(c.dataset(0).maxhitspernode(), is(100)); - } - } - - private String getMaxhitsTuning() { - return "<tuning>" + - " <dispatch>" + - " <max-hits-per-partition>100</max-hits-per-partition>" + - " </dispatch>" + - "</tuning>"; - } - - - @Test - public void requireThatSearchCoverageIsSetInMultilevelSetup() throws Exception { - ContentCluster cr = createCluster(getSimpleDispatchXml() + getCoverage()); - Dispatch tld = cr.getSearch().getIndexed().getTLDs().get(0); - PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); - tld.getConfig(builder); - PartitionsConfig config = new PartitionsConfig(builder); - assertThat(config.dataset().size(), is(1)); - assertEquals(95.0, config.dataset(0).minimal_searchcoverage(), 0.1); - for (Dispatch dispatch : getDispatchers(tld)) { - PartitionsConfig.Builder b = new PartitionsConfig.Builder(); - dispatch.getConfig(b); - PartitionsConfig c= new PartitionsConfig(b); - assertThat(c.dataset().size(), is(1)); - assertEquals(95.0, c.dataset(0).minimal_searchcoverage(), 0.1); - } - } - - @Test - public void requireThatSearchCoverageIsSetInSingleLevelSetup() { - TestRoot root = new TestDriver(true).buildModel(new MockApplicationPackage.Builder() - .withServices("<services version='1.0'>" + - "<content id='stateful' version='1.0'>" + - " <redundancy>1</redundancy>" + - " <documents><document mode='index' type='music' /></documents>" + - " <nodes>" + - " <node distribution-key='1' hostalias='mockroot' />" + - " </nodes>" + - " <search><coverage><minimum>0.95</minimum></coverage></search>" + - "</content>" + - "<container id='foo' version='1.0'>" + - " <search />" + - " <nodes><node hostalias='mockroot' /></nodes>" + - "</container>" + - "</services>") - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()); - PartitionsConfig config = root.getConfig(PartitionsConfig.class, "stateful/search/cluster.stateful/tlds/foo.0.tld.0"); - assertThat(config.dataset().size(), is(1)); - assertEquals(95.0, config.dataset(0).minimal_searchcoverage(), 0.1); - } - - private String getCoverage() { - return "<search>" + - " <coverage>" + - " <minimum>0.95</minimum>" + - " </coverage>" + - "</search>"; - } - - @Test - public void requireThatDispatchGroupsCanBeExplicitlySpecified() throws Exception { - ContentCluster cr = createCluster(getDispatchXml()); - IndexedSearchCluster ix = cr.getSearch().getIndexed(); - Dispatch tld = cr.getSearch().getIndexed().getTLDs().get(0); - - assertEquals("tlds/tld.0", tld.getConfigId()); - assertEquals(0, getFdispatchrcConfig(tld).dispatchlevel()); - new EngineAsserter(2, 6, tld). - assertEngine(0, 0, "tcp/mh0:19113"). - assertEngine(1, 0, "tcp/mh2:19113"). - assertEngine(2, 0, "tcp/mh4:19113"). - assertEngine(0, 1, "tcp/mh1:19113"). - assertEngine(1, 1, "tcp/mh3:19113"). - assertEngine(2, 1, "tcp/mh5:19113"); - - List<Dispatch> ds = getDispatchers(tld); - assertEquals(6, ds.size()); - { // dispatch group 1 - Dispatch[] dispatchers = {ds.get(0), ds.get(1), ds.get(2)}; - String[] specs = {"tcp/mh0:19104", "tcp/mh2:19104", "tcp/mh4:19104"}; - SearchNode[] searchNodes = {ix.getSearchNode(0), ix.getSearchNode(2), ix.getSearchNode(4)}; - assertDispatchAndSearchNodes(0, dispatchers, specs, searchNodes); - } - { // dispatch group 2 - Dispatch[] dispatchers = {ds.get(3), ds.get(4), ds.get(5)}; - String[] specs = {"tcp/mh1:19104", "tcp/mh3:19104", "tcp/mh5:19104"}; - SearchNode[] searchNodes = {ix.getSearchNode(1), ix.getSearchNode(3), ix.getSearchNode(5)}; - assertDispatchAndSearchNodes(1, dispatchers, specs, searchNodes); - } - } - - @Test - public void requireThatUnevenDispatchGroupsCanBeCreated() { - List<SearchNode> searchNodes = createSearchNodes(5); - List<DispatchSpec.Group> groups = DispatchGroupBuilder.createDispatchGroups(searchNodes, 3); - assertEquals(3, groups.size()); - assertGroup(new int[]{0, 1}, groups.get(0)); - assertGroup(new int[]{2, 3}, groups.get(1)); - assertGroup(new int[]{4}, groups.get(2)); - } - - private List<SearchNode> createSearchNodes(int numNodes) { - List<SearchNode> searchNodes = new ArrayList<>(); - MockRoot root = new MockRoot(""); - for (int i = 0; i < numNodes; ++i) { - searchNodes.add(SearchNode.create(root, "mynode" + i, i, new NodeSpec(0, i), "mycluster", null, false, - Optional.empty(), Optional.empty(), root.getDeployState().isHosted())); - } - return searchNodes; - } - - private void assertGroup(int[] nodes, DispatchSpec.Group group) { - assertEquals(nodes.length, group.getNodes().size()); - for (int i = 0; i < nodes.length; ++i) { - assertEquals(nodes[i], group.getNodes().get(i).getDistributionKey()); - } - } - - private ContentCluster createIllegalSetupWithMultipleNodeReferences() throws Exception { - String dispatchXml = " <dispatch>\n" + - " <group>\n" + - " <node distribution-key='10'/>\n" + - " <node distribution-key='11'/>\n" + - " <node distribution-key='12'/>\n" + - " </group>\n" + - " <group>\n" + - " <node distribution-key='12'/>\n" + - " <node distribution-key='13'/>\n" + - " <node distribution-key='14'/>\n" + - " </group>\n" + - " </dispatch>\n"; - return createCluster(dispatchXml); - } - - private ContentCluster createIllegalSetupWithMissingNodeReferences() throws Exception { - String dispatchXml = " <dispatch>\n" + - " <group>\n" + - " <node distribution-key='10'/>\n" + - " <node distribution-key='11'/>\n" + - " </group>\n" + - " <group>\n" + - " <node distribution-key='13'/>\n" + - " <node distribution-key='14'/>\n" + - " </group>\n" + - " </dispatch>\n"; - return createCluster(dispatchXml); - } - - private ContentCluster createIllegalSetupWithIllegalNodeReference() throws Exception { - String dispatchXml = " <dispatch>\n" + - " <group>\n" + - " <node distribution-key='10'/>\n" + - " <node distribution-key='11'/>\n" + - " <node distribution-key='12'/>\n" + - " </group>\n" + - " <group>\n" + - " <node distribution-key='13'/>\n" + - " <node distribution-key='14'/>\n" + - " <node distribution-key='15'/>\n" + - " <node distribution-key='19'/>\n" + - " </group>\n" + - " </dispatch>\n"; - return createCluster(dispatchXml); - } - - @Test - public void requireThatWeReferenceNodesOnlyOnceWhenSettingUpDispatchGroups() { - try { - createIllegalSetupWithMultipleNodeReferences(); - assertFalse("Did not get expected Exception", true); - } catch (Exception e) { - assertThat(e.getMessage(), containsString("node with distribution key '12' is referenced multiple times")); - } - } - - @Test - public void requireThatWeReferenceAllNodesWhenSettingUpDispatchGroups() { - try { - createIllegalSetupWithMissingNodeReferences(); - assertFalse("Did not get expected Exception", true); - } catch (Exception e) { - assertThat(e.getMessage(), containsString("2 node(s) with distribution keys [12, 15] are not referenced")); - } - } - - @Test - public void requireThatWeReferenceValidNodesWhenSettingUpDispatchGroups() { - try { - createIllegalSetupWithIllegalNodeReference(); - assertFalse("Did not get expected Exception", true); - } catch (Exception e) { - assertThat(e.getMessage(), containsString("node with distribution key '19' does not exists")); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java deleted file mode 100644 index 227ad9c6be1..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.model.test.TestDriver; -import com.yahoo.vespa.config.search.core.PartitionsConfig; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Simon Thoresen Hult - */ -public class TldTest { - - @Test - public void requireThatServicesIsParsed() { - ApplicationPackage app = new MockApplicationPackage.Builder() - .withHosts("<hosts><host name='localhost'><alias>mockhost</alias></host><host name='my.other.host'><alias>mockhost2</alias></host></hosts>") - .withServices( - "<services>" + - " <admin version='2.0'>" + - " <adminserver hostalias='mockhost' />" + - " </admin>" + - " <container version='1.0' id='default'>" + - " <search />" + - " <nodes>" + - " <node hostalias='mockhost'/>" + - " </nodes>" + - " </container>" + - " <content version='1.0' id='foo'>" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type='music' mode='index'/>" + - " </documents>" + - " <group>" + - " <node hostalias='mockhost' distribution-key='0'/>" + - " <node hostalias='mockhost2' distribution-key='1'/>" + - " </group>" + - " <tuning>" + - " <dispatch>" + - " <max-hits-per-partition>69</max-hits-per-partition>" + - " <use-local-node>true</use-local-node>" + - " </dispatch>" + - " </tuning>" + - " </content>" + - "</services>") - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build(); - - PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); - new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.0.tld.0"); - PartitionsConfig config = new PartitionsConfig(builder); - - assertEquals(1, config.dataset().size()); - assertEquals(69, config.dataset(0).maxhitspernode()); - assertEquals(1, config.dataset(0).engine().size()); - } - - @Test - public void requireThatUseLocalPolicyIsOk() { - ApplicationPackage app = new MockApplicationPackage.Builder() - .withHosts( - "<hosts>" + - " <host name='search.node1'><alias>search1</alias></host>" + - " <host name='search.node2'><alias>search2</alias></host>" + - " <host name='jdisc.host.other'><alias>gateway</alias></host>" + - "</hosts>") - .withServices( - "<services>" + - " <admin version='2.0'>" + - " <adminserver hostalias='gateway' />" + - " </admin>" + - " <container version='1.0' id='default'>" + - " <search />" + - " <nodes>" + - " <node hostalias='search1'/>" + - " <node hostalias='search2'/>" + - " </nodes>" + - " </container>" + - " <container version='1.0' id='gw'>" + - " <document-api/>" + - " <nodes>" + - " <node hostalias='gateway'/>" + - " </nodes>" + - " </container>" + - " <content version='1.0' id='foo'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='music' mode='index'/>" + - " </documents>" + - " <group name='topGroup'>" + - " <distribution partitions='1|*'/>" + - " <group name='group1' distribution-key='0'>" + - " <node hostalias='search1' distribution-key='0'/>" + - " </group>" + - " <group name='group2' distribution-key='1'>" + - " <node hostalias='search2' distribution-key='1'/>" + - " </group>" + - " </group>" + - " <tuning>" + - " <dispatch>" + - " <use-local-node>true</use-local-node>" + - " </dispatch>" + - " </tuning>" + - " </content>" + - "</services>") - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build(); - - PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); - new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/gw.0.tld.0"); - PartitionsConfig config = new PartitionsConfig(builder); - - // No tld if no search - assertEquals(0, config.dataset().size()); - - // First container with a local search node - builder = new PartitionsConfig.Builder(); - new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.0.tld.0"); - config = new PartitionsConfig(builder); - - assertEquals(1, config.dataset().size()); - assertEquals(1, config.dataset(0).engine().size()); - assertEquals(0,config.dataset(0).engine(0).rowid()); - assertEquals(0,config.dataset(0).engine(0).partid()); - assertTrue("Configured with local search node as engine", - config.dataset(0).engine(0).name_and_port().contains("search.node1")); - - // Second container with a local search node - builder = new PartitionsConfig.Builder(); - new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.1.tld.1"); - config = new PartitionsConfig(builder); - - assertEquals(1, config.dataset().size()); - assertEquals(1, config.dataset(0).engine().size()); - - assertEquals("rowid equals 0",0, config.dataset(0).engine(0).rowid()); // Load Balance row 0 - assertEquals("partid equals 0",0, config.dataset(0).engine(0).partid()); - assertTrue("Configured with correct search node", - config.dataset(0).engine(0).name_and_port().contains("search.node2")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/utils/DispatchUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/search/utils/DispatchUtils.java deleted file mode 100644 index 0185770037f..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/utils/DispatchUtils.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search.utils; - -import com.yahoo.vespa.config.search.core.FdispatchrcConfig; -import com.yahoo.vespa.config.search.core.PartitionsConfig; -import com.yahoo.vespa.model.search.Dispatch; - -import static org.junit.Assert.assertEquals; - -public class DispatchUtils { - - public static PartitionsConfig.Dataset getDataset(Dispatch dispatch) { - PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); - dispatch.getConfig(builder); - PartitionsConfig cfg = new PartitionsConfig(builder); - assertEquals(1, cfg.dataset().size()); - return cfg.dataset(0); - } - - public static FdispatchrcConfig getFdispatchrcConfig(Dispatch dispatch) { - FdispatchrcConfig.Builder builder = new FdispatchrcConfig.Builder(); - dispatch.getConfig(builder); - return new FdispatchrcConfig(builder); - } - - public static void assertEngine(int rowId, int partitionId, PartitionsConfig.Dataset.Engine engine) { - assertEquals(rowId, engine.rowid()); - assertEquals(partitionId, engine.partid()); - } - - public static void assertEngine(int rowId, int partitionId, String connectSpec, PartitionsConfig.Dataset.Engine engine) { - assertEngine(rowId, partitionId, engine); - assertEquals(connectSpec, engine.name_and_port()); - } -} diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index 84d5c032395..5229e7da0b8 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -16,6 +16,13 @@ maxNodesDownPerGroup int default=0 # Distribution policy for group selection distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ROUNDROBIN +## Maximum number of hits that will be requested from a single node +## in this dataset. If not set, there is no limit. Using this option +## may help reduce network traffic when searching in datasets with big +## fan-out, but it will also result in incorrect and incomplete results; +## don't use it if you don't (really) mean it. +maxHitsPerNode int default=2147483647 + # Is multi-level dispatch configured for this cluster # Deprecated, will go away soon, NOOP useMultilevelDispatch bool default=false diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 33c568061fe..2daa371db8f 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,6 +7,10 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true +# Scale upstream connection cache with number of upstream servers +tenants{}.applications{}.dynamicUpstreamConnectionCache bool default=false +# Send health check requests on port 4081 instead of 4080 +tenants{}.applications{}.healthCheckOn4081 bool default=false tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 0e17a31886b..3a9e5a55191 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -5,7 +5,6 @@ import com.google.common.io.Files; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; @@ -71,8 +70,10 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -326,7 +327,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Deployment deployment = deployFromPreparedSession(localSession, tenant, timeoutBudget.timeLeft()); deployment.setIgnoreSessionStaleFailure(ignoreSessionStaleFailure); deployment.activate(); - return localSession.getApplicationId(); + ApplicationId applicationId = localSession.getApplicationId(); + return applicationId; } private Deployment deployFromPreparedSession(LocalSession session, Tenant tenant, Duration timeout) { @@ -444,6 +446,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return fileReferencesToDelete; } + public Set<FileReference> getFileReferences(ApplicationId applicationId) { + return getOptionalApplication(applicationId).map(app -> app.getModel().fileReferences()).orElse(Set.of()); + } + public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) { Tenant tenant = tenantRepository.getTenant(tenantName); return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index b6e1d1873c9..9e81d3c0525 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; import com.yahoo.log.LogLevel; @@ -120,14 +121,12 @@ public class Deployment implements com.yahoo.config.provision.Deployment { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - try (Lock lock = tenant.getApplicationRepo().lock(session.getApplicationId())) { + ApplicationId applicationId = session.getApplicationId(); + try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { validateSessionStatus(session); NestedTransaction transaction = new NestedTransaction(); - transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(session.getApplicationId()), session, ignoreSessionStaleFailure)); - - if (hostProvisioner.isPresent()) { - hostProvisioner.get().activate(transaction, session.getApplicationId(), session.getAllocatedHosts().getHosts()); - } + transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(applicationId), session, ignoreSessionStaleFailure)); + hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); transaction.commit(); } catch (RuntimeException e) { throw e; @@ -139,8 +138,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + - ( hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner" ) + - ". Config generation " + session.getMetaData().getGeneration()); + (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") + + ". Config generation " + session.getMetaData().getGeneration() + + ". File references used: " + applicationRepository.getFileReferences(applicationId)); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 48f5618886b..9cba549838e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -131,7 +131,6 @@ public class ModelContextImpl implements ModelContext { private final Set<ContainerEndpoint> endpoints; private final boolean isBootstrap; private final boolean isFirstTimeDeployment; - private final boolean useDedicatedNodeForLogserver; private final boolean useAdaptiveDispatch; private final Optional<TlsSecrets> tlsSecrets; private final double defaultTermwiseLimit; @@ -162,8 +161,6 @@ public class ModelContextImpl implements ModelContext { this.endpoints = endpoints; this.isBootstrap = isBootstrap; this.isFirstTimeDeployment = isFirstTimeDeployment; - this.useDedicatedNodeForLogserver = Flags.USE_DEDICATED_NODE_FOR_LOGSERVER.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.tlsSecrets = tlsSecrets; @@ -212,9 +209,6 @@ public class ModelContextImpl implements ModelContext { public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } @Override - public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } - - @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 5dd2c8fd58a..3610de172a2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,7 +9,10 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -32,10 +35,14 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; + private final BooleanFlag dynamicUpstreamConnectionCache; + private final BooleanFlag healthCheckOn4081; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; + this.dynamicUpstreamConnectionCache = Flags.DYNAMIC_UPSTREAM_CONNECTION_CACHE.bindTo(flagSource); + this.healthCheckOn4081 = Flags.HEALTH_CHECK_ON_4081.bindTo(flagSource); } @Override @@ -62,6 +69,8 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); + ab.dynamicUpstreamConnectionCache(dynamicUpstreamConnectionCache.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value()); + ab.healthCheckOn4081(healthCheckOn4081.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value()); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 860bbdc134c..3102163df0d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -81,6 +81,5 @@ public class ModelContextImplTest { assertThat(context.properties().rotations(), equalTo(rotations)); assertThat(context.properties().endpoints(), equalTo(endpoints)); assertThat(context.properties().isFirstTimeDeployment(), equalTo(false)); - assertThat(context.properties().useDedicatedNodeForLogserver(), equalTo(true)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index bc777659298..d490f24bed4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -17,6 +17,7 @@ import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; @@ -110,6 +111,22 @@ public class LbServicesProducerTest { } } + @Test + public void dynamic_upstream_connections_is_configured_from_feature_flag() throws IOException, SAXException { + { + flagSource.withBooleanFlag(Flags.DYNAMIC_UPSTREAM_CONNECTION_CACHE.id(), true); + RegionName regionName = RegionName.from("us-east-1"); + LbServicesConfig conf = createModelAndGetLbServicesConfig(regionName); + assertTrue(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").dynamicUpstreamConnectionCache()); + } + { + flagSource.withBooleanFlag(Flags.DYNAMIC_UPSTREAM_CONNECTION_CACHE.id(), false); + RegionName regionName = RegionName.from("us-east-2"); + LbServicesConfig conf = createModelAndGetLbServicesConfig(regionName); + assertFalse(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").dynamicUpstreamConnectionCache()); + } + } + private LbServicesConfig createModelAndGetLbServicesConfig(RegionName regionName) throws IOException, SAXException { Zone zone = new Zone(Environment.prod, regionName); Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() diff --git a/container-core/src/main/resources/configdefinitions/qr-searchers.def b/container-core/src/main/resources/configdefinitions/qr-searchers.def index bb8f80052f1..1e8bd54acc3 100644 --- a/container-core/src/main/resources/configdefinitions/qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/qr-searchers.def @@ -71,10 +71,3 @@ searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME ## Storage cluster route to use for search cluster if indexingmode is streaming. searchcluster[].storagecluster.routespec string default="" - -# The available dispatchers on each search cluster -searchcluster[].dispatcher[].host string -searchcluster[].dispatcher[].port int - -# Per dispatcher config-id might be nice to have, remove it until needed. -# searchcluster[].dispatcher[].configid reference diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 22af77020eb..1f621eb926c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -23,7 +23,6 @@ import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; import org.apache.commons.lang.StringUtils; @@ -107,17 +106,9 @@ public class ClusterSearcher extends Searcher { vipStatus.addToRotation(searcher.getName()); } else { Dispatcher dispatcher = Dispatcher.create(id.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus, metric); - for (int dispatcherIndex = 0; dispatcherIndex < searchClusterConfig.dispatcher().size(); dispatcherIndex++) { - try { - if ( ! isRemote(searchClusterConfig.dispatcher(dispatcherIndex).host())) { - FastSearcher searcher = searchDispatch(searchClusterIndex, fs4ResourcePool.getServerId(), docSumParams, - documentDbConfig, dispatcher, dispatcherIndex); - addBackendSearcher(searcher); - } - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } - } + FastSearcher searcher = searchDispatch(searchClusterIndex, fs4ResourcePool.getServerId(), docSumParams, documentDbConfig, dispatcher); + addBackendSearcher(searcher); + } if ( server == null ) { throw new IllegalStateException("ClusterSearcher should have backend."); @@ -133,26 +124,16 @@ public class ClusterSearcher extends Searcher { return null; } - /** - * Returns false if this host is local. - */ - boolean isRemote(String host) throws UnknownHostException { - return (InetAddress.getByName(host).isLoopbackAddress()) - ? false - : !host.equals(HostName.getLocalhost()); - } - - private static ClusterParams makeClusterParams(int searchclusterIndex, int dispatchIndex) { - return new ClusterParams("sc" + searchclusterIndex + ".num" + dispatchIndex); + private static ClusterParams makeClusterParams(int searchclusterIndex) { + return new ClusterParams("sc" + searchclusterIndex + ".num" + 0); } private static FastSearcher searchDispatch(int searchclusterIndex, String serverId, SummaryParameters docSumParams, DocumentdbInfoConfig documentdbInfoConfig, - Dispatcher dispatcher, - int dispatcherIndex) { - ClusterParams clusterParams = makeClusterParams(searchclusterIndex, dispatcherIndex); + Dispatcher dispatcher) { + ClusterParams clusterParams = makeClusterParams(searchclusterIndex); return new FastSearcher(serverId, dispatcher, docSumParams, clusterParams, documentdbInfoConfig); } @@ -164,7 +145,7 @@ public class ClusterSearcher extends Searcher { if (searchClusterConfig.searchdef().size() != 1) { throw new IllegalArgumentException("Search clusters in streaming search shall only contain a single searchdefinition : " + searchClusterConfig.searchdef()); } - ClusterParams clusterParams = makeClusterParams(searchclusterIndex, 0); + ClusterParams clusterParams = makeClusterParams(searchclusterIndex); VdsStreamingSearcher searcher = new VdsStreamingSearcher(); searcher.setSearchClusterConfigId(searchClusterConfig.rankprofiles().configid()); searcher.setDocumentType(searchClusterConfig.searchdef(0)); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java index ee64632d40a..93a21476f35 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java @@ -10,6 +10,7 @@ import java.util.Comparator; import java.util.List; public class SortDataHitSorter { + public static void sort(HitGroup hitGroup, List<Hit> hits) { var sorting = hitGroup.getQuery().getRanking().getSorting(); var fallbackOrderer = hitGroup.getOrderer(); @@ -61,4 +62,5 @@ public class SortDataHitSorter { return fallback.compare(left, right); } } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java b/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java index 2a1e3199d7d..30bf0ff4bb0 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java @@ -14,7 +14,6 @@ import com.yahoo.search.result.Relevance; */ public class Group extends HitGroup { - private static final long serialVersionUID = 2122928012157537800L; private final GroupId groupId; /** @@ -80,4 +79,5 @@ public class Group extends HitGroup { } return null; } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java b/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java index 03e0f7e6d59..653fc4cb978 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java @@ -6,20 +6,21 @@ import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.result.Hit; /** - * <p>This class represents a labeled hit list in the grouping result model. It is contained in {@link Group}, and + * This class represents a labeled hit list in the grouping result model. It is contained in {@link Group}, and * contains one or more {@link Hit hits} itself, making this the parent of leaf nodes in the hierarchy of grouping - * results. Use the {@link GroupingRequest#getResultGroup(Result)} to retrieve grouping results.</p> + * results. Use the {@link GroupingRequest#getResultGroup(Result)} to retrieve grouping results. * * @author Simon Thoresen Hult */ public class HitList extends AbstractList { /** - * <p>Constructs a new instance of this class.</p> + * Constructs a new instance of this class. * - * @param label The label to assign to this. + * @param label the label to assign to this */ public HitList(String label) { super("hitlist", label); } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java b/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java index 3907f87a276..da115807fc2 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java @@ -11,7 +11,7 @@ import java.util.Arrays; import java.util.Map; /** - * This is a helper class for rendering grouping results. + * A helper for rendering grouping results. * * @author Simon Thoresen Hult */ @@ -94,4 +94,5 @@ public abstract class HitRenderer { private static void renderContinuation(String id, Continuation continuation, XMLWriter writer) { writer.openTag(TAG_CONTINUATION).attribute(TAG_CONTINUATION_ID, id).content(continuation, false).closeTag(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java index e8f4d566028..5ea0e3a0838 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java @@ -44,7 +44,8 @@ class HitConverter implements ResultBuilder.HitConverter { } private Hit convertFs4Hit(String summaryClass, FS4Hit groupHit) { - FastHit hit = new FastHit(groupHit.getGlobalId().getRawId(), new Relevance(groupHit.getRank()), + FastHit hit = new FastHit(groupHit.getGlobalId().getRawId(), + new Relevance(groupHit.getRank()), groupHit.getPath(), groupHit.getDistributionKey()); hit.setFillable(); hit.setSearcherSpecificMetaData(searcher, summaryClass); @@ -72,4 +73,5 @@ class HitConverter implements ResultBuilder.HitConverter { } return ret; } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java index 2402be27b9c..f45617d1cd7 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java @@ -285,6 +285,7 @@ class ResultBuilder { page.putContinuations(hitList.continuations()); return hitList; } + } private class GroupListBuilder { @@ -341,6 +342,7 @@ class ResultBuilder { } return ret; } + } private class PageInfo { @@ -381,16 +383,17 @@ class ResultBuilder { } } } + } /** - * Defines a helper interface to convert Vespa style grouping hits into corresponding instances of {@link Hit}. It - * is an interface to simplify testing. - * - * @author Simon Thoresen Hult + * Defines a helper interface to convert Vespa style grouping hits into corresponding instances of {@link Hit}. + * It is an interface to simplify testing. */ public interface HitConverter { - public com.yahoo.search.result.Hit toSearchHit(String summaryClass, com.yahoo.searchlib.aggregation.Hit hit); + com.yahoo.search.result.Hit toSearchHit(String summaryClass, com.yahoo.searchlib.aggregation.Hit hit); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 64db1cf0062..1c58081e4f1 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -394,27 +394,29 @@ public class SearchHandler extends LoggingRequestHandler { log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); return new Result(query, error); } catch (IllegalArgumentException e) { - ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " - + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); - return new Result(query, error); - } catch (LinkageError e) { - // Should have been an Exception in an OSGi world - typical bundle dependency issue problem - ErrorMessage error = ErrorMessage.createErrorInPluginSearcher( - "Error executing " + searchChain + "]: " + Exceptions.toMessageString(e), e); - log(request, query, e); - return new Result(query, error); - } catch (StackOverflowError e) { // Also recoverable - ErrorMessage error = ErrorMessage.createErrorInPluginSearcher( - "Error executing " + searchChain + "]: " + Exceptions.toMessageString(e), e); + if ("Comparison method violates its general contract!".equals(e.getMessage())) { + // This is an error in application components or Vespa code + log(request, query, e); + return new Result(query, ErrorMessage.createUnspecifiedError("Failed searching: " + + Exceptions.toMessageString(e), e)); + } + else { + ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " + + Exceptions.toMessageString(e)); + log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); + return new Result(query, error); + } + } catch (LinkageError | StackOverflowError e) { + // LinkageError should have been an Exception in an OSGi world - typical bundle dependency issue problem + // StackOverflowError is recoverable + ErrorMessage error = ErrorMessage.createErrorInPluginSearcher("Error executing " + searchChain + "]: " + + Exceptions.toMessageString(e), e); log(request, query, e); return new Result(query, error); } catch (Exception e) { - Result result = new Result(query); log(request, query, e); - result.hits().addError( - ErrorMessage.createUnspecifiedError("Failed searching: " + Exceptions.toMessageString(e), e)); - return result; + return new Result(query, ErrorMessage.createUnspecifiedError("Failed searching: " + + Exceptions.toMessageString(e), e)); } } @@ -448,12 +450,10 @@ public class SearchHandler extends LoggingRequestHandler { private void log(String request, Query query, Throwable e) { // Attempted workaround for missing stack traces if (e.getStackTrace().length == 0) { - log.log(LogLevel.ERROR, - "Failed executing " + query.toDetailString() + " [" + request - + "], received exception with no context", e); + log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + + " [" + request + "], received exception with no context", e); } else { - log.log(LogLevel.ERROR, - "Failed executing " + query.toDetailString() + " [" + request + "]", e); + log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + " [" + request + "]", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/result/HitOrderer.java b/container-search/src/main/java/com/yahoo/search/result/HitOrderer.java index 331a9ab5852..848666d628b 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitOrderer.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitOrderer.java @@ -9,15 +9,13 @@ import java.util.List; * * @author bratseth */ - public abstract class HitOrderer { /** Orders the given list of hits */ public abstract void order(List<Hit> hits); /** - * Returns the Comparator that this HitOrderer uses internally to - * sort hits. Returns null if no Comparator is used. + * Returns the Comparator that this HitOrderer uses internally to sort hits. Returns null if no Comparator is used. * <p> * This default implementation returns null. * diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index b660d90072b..dfc603846fd 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -1039,7 +1039,7 @@ public class YqlParser implements Parser { assertHasOperator(ast, ExpressionOperator.CONTAINS); String field = getIndex(ast.getArgument(0)); if (userQuery != null && indexFactsSession.getIndex(field).isAttribute()) { - userQuery.trace("Field '" + field + "' is an attribute, 'contains' will only match exactly", 1); + userQuery.trace("Field '" + field + "' is an attribute, 'contains' will only match exactly", 2); } return instantiateLeafItem(field, ast.<OperatorNode<ExpressionOperator>> getArgument(1)); } diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index c772300876b..f35d77de01a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -21,12 +21,9 @@ import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; import com.yahoo.vespa.config.search.DispatchConfig; import org.junit.Test; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -420,26 +417,6 @@ public class ClusterSearcherTestCase { } @Test - public void testLocalConnect() throws UnknownHostException { - ClusterSearcher cluster = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("dummy"))); - boolean canFindYahoo; - final String yahoo = "www.yahoo.com"; - - try { - canFindYahoo = (null != InetAddress.getByName(yahoo)); - } catch (Exception e) { - canFindYahoo = false; - } - - assertFalse(cluster.isRemote("127.0.0.1")); - assertFalse(cluster.isRemote("localhost")); - - if (canFindYahoo) { - assertTrue(cluster.isRemote(yahoo)); - } - } - - @Test public void testRequireThatSearchFailsForUndefinedRankProfileWithOneDocType() { Execution execution = createExecution(Arrays.asList("type1"), false); @@ -525,10 +502,6 @@ public class ClusterSearcherTestCase { searchClusterConfig.searchdef("streaming_sd"); } qrSearchersConfig.searchcluster(searchClusterConfig); - QrSearchersConfig.Searchcluster.Dispatcher.Builder dispatcherConfig = new QrSearchersConfig.Searchcluster.Dispatcher.Builder(); - dispatcherConfig.host("localhost"); - dispatcherConfig.port(0); - searchClusterConfig.dispatcher(dispatcherConfig); ClusterConfig.Builder clusterConfig = new ClusterConfig.Builder().clusterName(clusterName); if (maxQueryTimeout != null) diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index 2e5cfab36cc..ed623c82259 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -170,6 +170,9 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { .withSecurityOpts(new ArrayList<>(securityOpts)) .withBinds(volumeBinds) .withUlimits(ulimits) + // Docker version 1.13.1 patch 94 changed default pids.max for the Docker container's cgroup + // from max to 4096. -1L reinstates "max". File: /sys/fs/cgroup/pids/docker/CONTAINERID/pids.max. + .withPidsLimit(-1L) .withCapAdd(addCapabilities.toArray(new Capability[0])) .withCapDrop(dropCapabilities.toArray(new Capability[0])) .withPrivileged(privileged); @@ -240,6 +243,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { toOptionalOption("--memory", containerResources.map(ContainerResources::memoryBytes)), toRepeatedOption("--label", labelList), toRepeatedOption("--ulimit", ulimitList), + "--pids-limit -1", toRepeatedOption("--env", environmentAssignments), toRepeatedOption("--volume", volumeBindSpecs), toRepeatedOption("--cap-add", addCapabilitiesList), diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java index 7d41d873be2..3b8b0a84e64 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImplTest.java @@ -49,6 +49,7 @@ public class CreateContainerCommandImplTest { "--label my-label=test-label " + "--ulimit nofile=1:2 " + "--ulimit nproc=10:20 " + + "--pids-limit -1 " + "--env env1=val1 " + "--env env2=val2 " + "--volume vol1:/host/vol1:Z " + diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index 08b0fe94046..a616c853bf4 100755 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java index a0bce3cfbf6..27327daab47 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -818,15 +818,15 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu } protected ValueUpdate readTensorModifyUpdate(DataType type) { - throw new DeserializationException("Cannot deserialize TensorModifyUpdate, not implemented for Vespa 6"); + throw new DeserializationException("Cannot deserialize TensorModifyUpdate, not implemented for this document format version"); } protected ValueUpdate readTensorAddUpdate(DataType type) { - throw new DeserializationException("Cannot deserialize TensorAddUpdate, not implemented for Vespa 6"); + throw new DeserializationException("Cannot deserialize TensorAddUpdate, not implemented for this document format version"); } protected ValueUpdate readTensorRemoveUpdate(DataType type) { - throw new DeserializationException("Cannot deserialize TensorRemoveUpdate, not implemented for Vespa 6"); + throw new DeserializationException("Cannot deserialize TensorRemoveUpdate, not implemented for this document format version"); } } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index b12e54a0f12..c2111edfd10 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -686,17 +686,17 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume @Override public void write(TensorModifyUpdate update) { - throw new IllegalArgumentException("Write of TensorModifyUpdate not implemented for Vespa 6"); + throw new IllegalArgumentException("Write of TensorModifyUpdate not implemented for this document format version"); } @Override public void write(TensorAddUpdate update) { - throw new IllegalArgumentException("Write of TensorAddUpdate not implemented for Vespa 6"); + throw new IllegalArgumentException("Write of TensorAddUpdate not implemented for this document format version"); } @Override public void write(TensorRemoveUpdate update) { - throw new IllegalArgumentException("Write of TensorRemoveUpdate not implemented for Vespa 6"); + throw new IllegalArgumentException("Write of TensorRemoveUpdate not implemented for this document format version"); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 8bb16e8f334..54c815b48a3 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -73,11 +73,6 @@ public class Flags { "Takes effect immediately (but used only during provisioning).", NODE_TYPE); - public static final UnboundBooleanFlag USE_DEDICATED_NODE_FOR_LOGSERVER = defineFeatureFlag( - "use-dedicated-node-for-logserver", true, - "Whether to use a dedicated node for the logserver.", "Takes effect at redeployment", - APPLICATION_ID); - public static final UnboundDoubleFlag CONTAINER_CPU_CAP = defineDoubleFlag( "container-cpu-cap", 0, "Hard limit on how many CPUs a container may use. This value is multiplied by CPU allocated to node, so " + @@ -158,6 +153,23 @@ public class Flags { "Takes effect on next tick or on host-admin restart (may vary where used).", HOSTNAME); + public static final UnboundBooleanFlag USE_INTERNAL_ZTS = defineFeatureFlag( + "use-internal-zts", false, + "Decides if certificate in public should be requested from 'zts' configserver or mapped in", + "Takes effect on next tick or on host-admin restart.", + APPLICATION_ID); + + public static final UnboundBooleanFlag DYNAMIC_UPSTREAM_CONNECTION_CACHE = defineFeatureFlag( + "dynamic-upstream-connection-cache", false, + "Scale upstream connection cache with number of upstream servers", + "Takes effect on routing container redeployment"); + + public static final UnboundBooleanFlag HEALTH_CHECK_ON_4081 = defineFeatureFlag( + "health-check-on-4081", false, + "Change nginx to send health check requests on port 4081 instead of 4080.", + "Takes effect on routing container redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index 4dfdbd55fab..e206cb3215d 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -136,6 +136,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { .setSslcontext(sslContextFactory.getSslContext()) .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) .setUserTokenHandler(context -> null) // https://stackoverflow.com/a/42112034/1615280 + .setUserAgent("health-check-proxy-client") .build(); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index bdcb96c1861..fa439e50fbd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; @@ -287,6 +288,24 @@ public final class Node { return this.with(newStatus).with(newHistory); } + /** Returns a copy of this node with the current OS version set to the given version at the given instant */ + public Node withCurrentOsVersion(Version version, Instant instant) { + var newStatus = status.withOsVersion(version); + var newHistory = history(); + // Only update history if version has changed + if (status.osVersion().isEmpty() || !status.osVersion().get().equals(version)) { + newHistory = history.with(new History.Event(History.Event.Type.osUpgraded, Agent.system, instant)); + } + return this.with(newStatus).with(newHistory); + } + + /** Returns a copy of this node with firmware verified at the given instant */ + public Node withFirmwareVerifiedAt(Instant instant) { + var newStatus = status.withFirmwareVerifiedAt(instant); + var newHistory = history.with(new History.Event(History.Event.Type.firmwareVerified, Agent.system, instant)); + return this.with(newStatus).with(newHistory); + } + /** Returns a copy of this node with the given history. */ public Node with(History history) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 8dcf3c260f5..5f4c300b496 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -120,9 +120,9 @@ public class NodePatcher { case "currentVespaVersion" : return node.with(node.status().withVespaVersion(Version.fromString(asString(value)))); case "currentOsVersion" : - return node.with(node.status().withOsVersion(Version.fromString(asString(value)))); + return node.withCurrentOsVersion(Version.fromString(asString(value)), clock.instant()); case "currentFirmwareCheck": - return node.with(node.status().withFirmwareVerifiedAt(Instant.ofEpochMilli(asLong(value)))); + return node.withFirmwareVerifiedAt(Instant.ofEpochMilli(asLong(value))); case "failCount" : return node.with(node.status().setFailCount(asLong(value).intValue())); case "flavor" : diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 23699879ceb..0cfdf80a8a1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -325,10 +325,17 @@ public class SerializationTest { assertFalse(serialized.status().osVersion().isPresent()); // Update OS version - serialized = serialized.with(serialized.status() - .withOsVersion(Version.fromString("7.1"))); + serialized = serialized.withCurrentOsVersion(Version.fromString("7.1"), Instant.ofEpochMilli(123)) + // Another update for same version: + .withCurrentOsVersion(Version.fromString("7.1"), Instant.ofEpochMilli(456)); serialized = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(serialized)); assertEquals(Version.fromString("7.1"), serialized.status().osVersion().get()); + var osUpgradedEvents = serialized.history().events().stream() + .filter(event -> event.type() == History.Event.Type.osUpgraded) + .collect(Collectors.toList()); + assertEquals("OS upgraded event is added", 1, osUpgradedEvents.size()); + assertEquals("Duplicate updates of same version uses earliest instant", Instant.ofEpochMilli(123), + osUpgradedEvents.get(0).at()); } @Test @@ -336,9 +343,10 @@ public class SerializationTest { Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); assertFalse(node.status().firmwareVerifiedAt().isPresent()); - node = node.with(node.status().withFirmwareVerifiedAt(Instant.ofEpochMilli(100))); + node = node.withFirmwareVerifiedAt(Instant.ofEpochMilli(100)); node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); assertEquals(100, node.status().firmwareVerifiedAt().get().toEpochMilli()); + assertEquals(Instant.ofEpochMilli(100), node.history().event(History.Event.Type.firmwareVerified).get().at()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json index f5068924084..31bdfe7d6e3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json @@ -57,6 +57,11 @@ "event": "activated", "at": 123, "agent": "application" + }, + { + "event": "osUpgraded", + "at": 123, + "agent": "system" } ], "ipAddresses": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json index 53982c78042..24e64248b1c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json @@ -57,6 +57,11 @@ "event": "activated", "at": 123, "agent": "application" + }, + { + "event": "firmwareVerified", + "at": 100, + "agent": "system" } ], "ipAddresses": [ diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index b687576a0f0..558914805d1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(searchcore_matching STATIC blueprintbuilder.cpp constant_value_repo.cpp docid_range_scheduler.cpp + docsum_matcher.cpp document_scorer.cpp fakesearchcontext.cpp handlerecorder.cpp @@ -19,7 +20,6 @@ vespa_add_library(searchcore_matching STATIC match_tools.cpp matcher.cpp matching_stats.cpp - unpacking_iterators_optimizer.cpp partial_result.cpp query.cpp queryenvironment.cpp @@ -35,6 +35,7 @@ vespa_add_library(searchcore_matching STATIC sessionmanager.cpp termdataextractor.cpp termdatafromnode.cpp + unpacking_iterators_optimizer.cpp viewresolver.cpp DEPENDS searchcore_grouping diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp new file mode 100644 index 00000000000..73aab5b3fca --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp @@ -0,0 +1,137 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "docsum_matcher.h" +#include <vespa/eval/eval/tensor.h> +#include <vespa/eval/eval/tensor_engine.h> +#include <vespa/vespalib/objects/nbostream.h> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.matching.docsum_matcher"); + +using search::FeatureSet; +using search::StructFieldMapper; +using search::MatchingElements; +using search::fef::RankProgram; +using search::fef::FeatureResolver; +using search::queryeval::SearchIterator; + +namespace proton::matching { + +namespace { + +FeatureSet::UP get_feature_set(const MatchToolsFactory &mtf, + const std::vector<uint32_t> &docs, + bool summaryFeatures) +{ + MatchTools::UP matchTools = mtf.createMatchTools(); + if (summaryFeatures) { + matchTools->setup_summary(); + } else { + matchTools->setup_dump(); + } + RankProgram &rankProgram = matchTools->rank_program(); + + std::vector<vespalib::string> featureNames; + FeatureResolver resolver(rankProgram.get_seeds(false)); + featureNames.reserve(resolver.num_features()); + for (size_t i = 0; i < resolver.num_features(); ++i) { + featureNames.emplace_back(resolver.name_of(i)); + } + auto retval = std::make_unique<FeatureSet>(featureNames, docs.size()); + if (docs.empty()) { + return retval; + } + FeatureSet &fs = *retval; + + SearchIterator &search = matchTools->search(); + search.initRange(docs.front(), docs.back()+1); + for (uint32_t i = 0; i < docs.size(); ++i) { + if (search.seek(docs[i])) { + uint32_t docId = search.getDocId(); + search.unpack(docId); + auto * f = fs.getFeaturesByIndex(fs.addDocId(docId)); + for (uint32_t j = 0; j < featureNames.size(); ++j) { + if (resolver.is_object(j)) { + auto obj = resolver.resolve(j).as_object(docId); + if (const auto *tensor = obj.get().as_tensor()) { + vespalib::nbostream buf; + tensor->engine().encode(*tensor, buf); + f[j].set_data(vespalib::Memory(buf.peek(), buf.size())); + } else { + f[j].set_double(obj.get().as_double()); + } + } else { + f[j].set_double(resolver.resolve(j).as_number(docId)); + } + } + } else { + LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. Skipping hit", docs[i]); + } + } + if (auto onSummaryTask = mtf.createOnSummaryTask()) { + onSummaryTask->run(docs); + } + return retval; +} + +} + +DocsumMatcher::DocsumMatcher() + : _from_session(), + _from_mtf(), + _mtf(nullptr), + _docs() +{ +} + +DocsumMatcher::DocsumMatcher(SearchSession::SP session, std::vector<uint32_t> docs) + : _from_session(std::move(session)), + _from_mtf(), + _mtf(&_from_session->getMatchToolsFactory()), + _docs(std::move(docs)) +{ +} + +DocsumMatcher::DocsumMatcher(MatchToolsFactory::UP mtf, std::vector<uint32_t> docs) + : _from_session(), + _from_mtf(std::move(mtf)), + _mtf(_from_mtf.get()), + _docs(std::move(docs)) +{ +} + +DocsumMatcher::~DocsumMatcher() { + if (_from_session) { + _from_session->releaseEnumGuards(); + } +} + +FeatureSet::UP +DocsumMatcher::get_summary_features() const +{ + if (!_mtf) { + return std::make_unique<FeatureSet>(); + } + return get_feature_set(*_mtf, _docs, true); +} + +FeatureSet::UP +DocsumMatcher::get_rank_features() const +{ + if (!_mtf) { + return std::make_unique<FeatureSet>(); + } + return get_feature_set(*_mtf, _docs, false); +} + +MatchingElements::UP +DocsumMatcher::get_matching_elements(const StructFieldMapper &field_mapper) const +{ + if (!_mtf) { + return std::make_unique<MatchingElements>(); + } + (void) field_mapper; + return std::make_unique<MatchingElements>(); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h new file mode 100644 index 00000000000..e1eb76be843 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h @@ -0,0 +1,45 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "search_session.h" +#include "match_tools.h" +#include <vespa/searchlib/common/featureset.h> +#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements.h> +#include <vector> +#include <memory> + +namespace proton::matching { + +/** + * Used to perform additional matching related to a docsum + * request. Note that external objects must be kept alive by the one + * using this class. + **/ +class DocsumMatcher +{ +private: + using FeatureSet = search::FeatureSet; + using StructFieldMapper = search::StructFieldMapper; + using MatchingElements = search::MatchingElements; + + SearchSession::SP _from_session; + MatchToolsFactory::UP _from_mtf; + MatchToolsFactory *_mtf; + std::vector<uint32_t> _docs; + +public: + DocsumMatcher(); + DocsumMatcher(SearchSession::SP session, std::vector<uint32_t> docs); + DocsumMatcher(MatchToolsFactory::UP mtf, std::vector<uint32_t> docs); + ~DocsumMatcher(); + + using UP = std::unique_ptr<DocsumMatcher>; + + FeatureSet::UP get_summary_features() const; + FeatureSet::UP get_rank_features() const; + MatchingElements::UP get_matching_elements(const StructFieldMapper &field_mapper) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index c2262cc51e5..7ebe05e7b96 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -117,59 +117,4 @@ MatchMaster::match(search::engine::Trace & trace, return reply; } -FeatureSet::SP -MatchMaster::getFeatureSet(const MatchToolsFactory &mtf, - const std::vector<uint32_t> &docs, bool summaryFeatures) -{ - MatchTools::UP matchTools = mtf.createMatchTools(); - if (summaryFeatures) { - matchTools->setup_summary(); - } else { - matchTools->setup_dump(); - } - RankProgram &rankProgram = matchTools->rank_program(); - - std::vector<vespalib::string> featureNames; - FeatureResolver resolver(rankProgram.get_seeds(false)); - featureNames.reserve(resolver.num_features()); - for (size_t i = 0; i < resolver.num_features(); ++i) { - featureNames.emplace_back(resolver.name_of(i)); - } - auto retval = std::make_shared<FeatureSet>(featureNames, docs.size()); - if (docs.empty()) { - return retval; - } - FeatureSet &fs = *retval; - - SearchIterator &search = matchTools->search(); - search.initRange(docs.front(), docs.back()+1); - for (uint32_t i = 0; i < docs.size(); ++i) { - if (search.seek(docs[i])) { - uint32_t docId = search.getDocId(); - search.unpack(docId); - auto * f = fs.getFeaturesByIndex(fs.addDocId(docId)); - for (uint32_t j = 0; j < featureNames.size(); ++j) { - if (resolver.is_object(j)) { - auto obj = resolver.resolve(j).as_object(docId); - if (const auto *tensor = obj.get().as_tensor()) { - vespalib::nbostream buf; - tensor->engine().encode(*tensor, buf); - f[j].set_data(vespalib::Memory(buf.peek(), buf.size())); - } else { - f[j].set_double(obj.get().as_double()); - } - } else { - f[j].set_double(resolver.resolve(j).as_number(docId)); - } - } - } else { - LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. Skipping hit", docs[i]); - } - } - if (auto onSummaryTask = mtf.createOnSummaryTask()) { - onSummaryTask->run(docs); - } - return retval; -} - } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.h b/searchcore/src/vespa/searchcore/proton/matching/match_master.h index c9a9a24945a..b88963da43c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.h @@ -32,9 +32,6 @@ public: uint32_t distributionKey, uint32_t numSearchPartitions); - static std::shared_ptr<search::FeatureSet> - getFeatureSet(const MatchToolsFactory &matchToolsFactory, - const std::vector<uint32_t> &docs, bool summaryFeatures); static MatchingStats getStats(MatchMaster && rhs) { return std::move(rhs._stats); } }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 72591d340a9..fe7a7616c18 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -58,20 +58,6 @@ struct StupidMetaStore : search::IDocumentMetaStore { void foreach(const search::IGidToLidMapperVisitor &) const override { } }; -FeatureSet::SP -findFeatureSet(const DocsumRequest &req, MatchToolsFactory &mtf, bool summaryFeatures) -{ - std::vector<uint32_t> docs; - docs.reserve(req.hits.size()); - for (const auto & hit : req.hits) { - if (hit.docid != search::endDocId) { - docs.push_back(hit.docid); - } - } - std::sort(docs.begin(), docs.end()); - return MatchMaster::getFeatureSet(mtf, docs, summaryFeatures); -} - size_t numThreads(size_t hits, size_t minHits) { return static_cast<size_t>(std::ceil(double(hits) / double(minHits))); } @@ -99,38 +85,6 @@ bool willNotNeedRanking(const SearchRequest & request, const GroupingContext & g } // namespace proton::matching::<unnamed> -FeatureSet::SP -Matcher::getFeatureSet(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, - SessionManager & sessionMgr, bool summaryFeatures) -{ - SessionId sessionId(&req.sessionId[0], req.sessionId.size()); - bool expectedSessionCached(false); - if (!sessionId.empty()) { - const Properties &cache_props = req.propertiesMap.cacheProperties(); - expectedSessionCached = cache_props.lookup("query").found(); - if (expectedSessionCached) { - SearchSession::SP session(sessionMgr.pickSearch(sessionId)); - if (session) { - MatchToolsFactory &mtf = session->getMatchToolsFactory(); - FeatureSet::SP result = findFeatureSet(req, mtf, summaryFeatures); - session->releaseEnumGuards(); - return result; - } - } - } - - StupidMetaStore metaStore; - MatchToolsFactory::UP mtf = create_match_tools_factory(req, searchCtx, attrCtx, metaStore, - req.propertiesMap.featureOverrides()); - if (!mtf->valid()) { - LOG(warning, "getFeatureSet(%s): query execution failed (%s). Returning empty feature set", - (summaryFeatures ? "summary features" : "rank features"), - (expectedSessionCached) ? "session has expired" : "invalid query"); - return std::make_shared<FeatureSet>(); - } - return findFeatureSet(req, *mtf, summaryFeatures); -} - Matcher::Matcher(const search::index::Schema &schema, const Properties &props, const vespalib::Clock &clock, QueryLimiter &queryLimiter, const IConstantValueRepo &constantValueRepo, uint32_t distributionKey) : _indexEnv(schema, props, constantValueRepo), @@ -360,27 +314,60 @@ FeatureSet::SP Matcher::getSummaryFeatures(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, SessionManager &sessionMgr) { - return getFeatureSet(req, searchCtx, attrCtx, sessionMgr, true); + auto docsum_matcher = create_docsum_matcher(req, searchCtx, attrCtx, sessionMgr); + return docsum_matcher->get_summary_features(); } FeatureSet::SP Matcher::getRankFeatures(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, SessionManager &sessionMgr) { - return getFeatureSet(req, searchCtx, attrCtx, sessionMgr, false); + auto docsum_matcher = create_docsum_matcher(req, searchCtx, attrCtx, sessionMgr); + return docsum_matcher->get_rank_features(); } -MatchingElements +MatchingElements::UP Matcher::get_matching_elements(const DocsumRequest &req, ISearchContext &search_ctx, IAttributeContext &attr_ctx, SessionManager &session_manager, const StructFieldMapper &field_mapper) { - (void) req; - (void) search_ctx; - (void) attr_ctx; - (void) session_manager; - (void) field_mapper; - return MatchingElements(); + auto docsum_matcher = create_docsum_matcher(req, search_ctx, attr_ctx, session_manager); + return docsum_matcher->get_matching_elements(field_mapper); +} + +DocsumMatcher::UP +Matcher::create_docsum_matcher(const DocsumRequest &req, ISearchContext &search_ctx, + IAttributeContext &attr_ctx, SessionManager &session_manager) +{ + std::vector<uint32_t> docs; + docs.reserve(req.hits.size()); + for (const auto &hit : req.hits) { + if (hit.docid != search::endDocId) { + docs.push_back(hit.docid); + } + } + std::sort(docs.begin(), docs.end()); + SessionId sessionId(&req.sessionId[0], req.sessionId.size()); + bool expectedSessionCached(false); + if (!sessionId.empty()) { + const Properties &cache_props = req.propertiesMap.cacheProperties(); + expectedSessionCached = cache_props.lookup("query").found(); + if (expectedSessionCached) { + SearchSession::SP session(session_manager.pickSearch(sessionId)); + if (session) { + return std::make_unique<DocsumMatcher>(std::move(session), std::move(docs)); + } + } + } + StupidMetaStore meta; + MatchToolsFactory::UP mtf = create_match_tools_factory(req, search_ctx, attr_ctx, meta, + req.propertiesMap.featureOverrides()); + if (!mtf->valid()) { + LOG(warning, "could not initialize docsum matching: %s", + (expectedSessionCached) ? "session has expired" : "invalid query"); + return std::make_unique<DocsumMatcher>(); + } + return std::make_unique<DocsumMatcher>(std::move(mtf), std::move(docs)); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index 3a1fad927b7..bbfcad04e15 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -7,6 +7,7 @@ #include "matching_stats.h" #include "search_session.h" #include "viewresolver.h" +#include "docsum_matcher.h" #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchlib/common/featureset.h> @@ -64,9 +65,6 @@ private: QueryLimiter &_queryLimiter; uint32_t _distributionKey; - search::FeatureSet::SP - getFeatureSet(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, - SessionManager &sessionMgr, bool summaryFeatures); std::unique_ptr<search::engine::SearchReply> handleGroupingSession(SessionManager &sessionMgr, search::grouping::GroupingContext & groupingContext, @@ -171,9 +169,12 @@ public: * about and how they relate to each other * @return matching elements **/ - MatchingElements get_matching_elements(const DocsumRequest &req, ISearchContext &search_ctx, - IAttributeContext &attr_ctx, SessionManager &session_manager, - const StructFieldMapper &field_mapper); + MatchingElements::UP get_matching_elements(const DocsumRequest &req, ISearchContext &search_ctx, + IAttributeContext &attr_ctx, SessionManager &session_manager, + const StructFieldMapper &field_mapper); + + DocsumMatcher::UP create_docsum_matcher(const DocsumRequest &req, ISearchContext &search_ctx, + IAttributeContext &attr_ctx, SessionManager &session_manager); /** * @return true if this rankprofile has summary-features enabled diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/AggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/AggregationResult.java index 264a9d4d4e9..6858ebb8f82 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/AggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/AggregationResult.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.objects.ObjectVisitor; import com.yahoo.vespa.objects.Serializer; /** - * <p>This is the aggregation super-class from which all types of aggregation inherits.</p> + * The result of some aggregation * * @author baldersheim * @author Simon Thoresen Hult @@ -20,7 +20,7 @@ public abstract class AggregationResult extends ExpressionNode { private int tag = -1; /** - * <p>Returns the tag of this aggregation result. This is useful for uniquely identifying a result.</p> + * Returns the tag of this aggregation result. This is useful for uniquely identifying a result. * * @return The numerical tag. */ @@ -29,10 +29,10 @@ public abstract class AggregationResult extends ExpressionNode { } /** - * <p>Assigns a tag to this group.</p> + * Assigns a tag to this group. * - * @param tag The numerical tag to set. - * @return This, to allow chaining. + * @param tag the numerical tag to set. + * @return this, to allow chaining. */ public AggregationResult setTag(int tag) { this.tag = tag; @@ -40,55 +40,47 @@ public abstract class AggregationResult extends ExpressionNode { } /** - * <p>This method is called when merging aggregation results. This method is simply a proxy for the abstract {@link - * #onMerge(AggregationResult)} method.</p> + * Called when merging aggregation results. This method is simply a proxy for the abstract {@link + * #onMerge(AggregationResult)} method. * - * @param result The result to merge with. + * @param result the result to merge with. */ public void merge(AggregationResult result) { onMerge(result); } /** - * <p>This method is called when all aggregation results have been merged. This method can be overloaded by - * subclasses that need special behaviour to occur after merge.</p> + * Hook called when all aggregation results have been merged. This method can be overloaded by + * subclasses that need special behaviour to occur after merge. */ public void postMerge() { // empty } - /** - * <p>This method returns a value that can be used for ranking.</p> - * - * @return The rankable result. - */ + /** Returns a value that can be used for ranking. */ public abstract ResultNode getRank(); /** - * <p>Sets the expression to aggregate on.</p> + * Sets the expression to aggregate on. * - * @param exp The expression. - * @return This, to allow chaining. + * @param exp the expression + * @return this, to allow chaining */ public AggregationResult setExpression(ExpressionNode exp) { expression = exp; return this; } - /** - * <p>Returns the expression to aggregate on.</p> - * - * @return The expression. - */ + /** Returns the expression to aggregate on. */ public ExpressionNode getExpression() { return expression; } /** - * <p>This method must be implemented by subclasses to support merge. It is called as the {@link - * #merge(AggregationResult)} method is invoked.</p> + * Mmust be implemented by subclasses to support merge. It is called as the {@link + * #merge(AggregationResult)} method is invoked. * - * @param result The result to merge with. + * @param result the result to merge with */ protected abstract void onMerge(AggregationResult result); @@ -158,4 +150,5 @@ public abstract class AggregationResult extends ExpressionNode { visitor.visit("expression", expression); visitor.visit("tag", tag); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java index 399ffd3128f..07de8bbdc55 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.objects.ObjectVisitor; import com.yahoo.vespa.objects.Serializer; /** - * This class represents a single hit from the fastserver4 backend + * A single hit from a Vespa content cluster * * @author havardpe */ @@ -103,7 +103,7 @@ public class FS4Hit extends Hit { return super.hashCode() + path + globalId.hashCode() + distributionKey; } - @SuppressWarnings({ "EqualsWhichDoesntCheckParameterClass", "RedundantIfStatement" }) + @SuppressWarnings("RedundantIfStatement") @Override public boolean equals(Object obj) { if (!super.equals(obj)) { @@ -129,4 +129,5 @@ public class FS4Hit extends Hit { visitor.visit("globalId", globalId.toString()); visitor.visit("distributionKey", distributionKey); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java index 73171f4dd00..c508296d739 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java @@ -12,28 +12,28 @@ public class Group extends Identifiable { public static final int classId = registerClass(0x4000 + 90, Group.class); private static final ObjectPredicate REF_LOCATOR = new RefLocator(); - private List<Integer> orderByIdx = new ArrayList<Integer>(); - private List<ExpressionNode> orderByExp = new ArrayList<ExpressionNode>(); - private List<AggregationResult> aggregationResults = new ArrayList<AggregationResult>(); - private List<Group> children = new ArrayList<Group>(); + private List<Integer> orderByIdx = new ArrayList<>(); + private List<ExpressionNode> orderByExp = new ArrayList<>(); + private List<AggregationResult> aggregationResults = new ArrayList<>(); + private List<Group> children = new ArrayList<>(); private ResultNode id = null; private double rank; private int tag = -1; private SortType sortType = SortType.UNSORTED; /** - * <p>This tells you if the children are ranked by the pure relevance or by a more complex expression. That - * indicates if the rank score from the child can be used for ordering.</p> + * This tells you if the children are ranked by the pure relevance or by a more complex expression. + * That indicates if the rank score from the child can be used for ordering. * - * @return True if it ranked by pure relevance. + * @return true if it ranked by pure relevance. */ public boolean isRankedByRelevance() { return orderByIdx.isEmpty(); } /** - * <p>Merges the content of the given group <b>into</b> this. When this function returns, make sure to call {@link - * #postMerge(java.util.List, int, int)}.</p> + * Merges the content of the given group <b>into</b> this. When this function returns, make sure to call + * {@link #postMerge(java.util.List, int, int)}. * * @param firstLevel The first level to merge. * @param currentLevel The current level. @@ -49,7 +49,7 @@ public class Group extends Identifiable { } } - ArrayList<Group> merged = new ArrayList<Group>(); + ArrayList<Group> merged = new ArrayList<>(); Iterator<Group> lhsChild = children.iterator(), rhsChild = rhs.children.iterator(); if (lhsChild.hasNext() && rhsChild.hasNext()) { Group lhsGroup = lhsChild.next(); @@ -93,8 +93,8 @@ public class Group extends Identifiable { } /** - * <p>After merging, this method will prune all levels so that they do not exceed the configured maximum number of - * groups per level.</p> + * After merging, this method will prune all levels so that they do not exceed the configured maximum number of + * groups per level. * * @param levels The specs of all grouping levels. * @param firstLevel The first level to merge. @@ -127,9 +127,7 @@ public class Group extends Identifiable { } - /** - * <p>Will sort the children by their id, if they are not sorted already.</p> - */ + /** Sorts the children by their id, if they are not sorted already. */ public void sortChildrenById() { if (sortType == SortType.BYID) { return; @@ -142,9 +140,7 @@ public class Group extends Identifiable { sortType = SortType.BYID; } - /** - * <p>Will sort the children by their rank, if they are not sorted already.</p> - */ + /** Sorts the children by their rank, if they are not sorted already. */ public void sortChildrenByRank() { if (sortType == SortType.BYRANK) { return; @@ -158,18 +154,16 @@ public class Group extends Identifiable { } /** - * <p>Returns the label to use for this group. See comment on {@link #setId(com.yahoo.searchlib.expression.ResultNode)} - * on the rationale of this being a {@link ResultNode}.</p> - * - * @return The label. + * Returns the label to use for this group. See comment on {@link #setId(com.yahoo.searchlib.expression.ResultNode)} + * on the rationale of this being a {@link ResultNode}. */ public ResultNode getId() { return id; } /** - * <p>Sets the label to use for this group. This is a {@link ResultNode} so that a group can be labeled with - * whatever value the classifier expression returns.</p> + * Sets the label to use for this group. This is a {@link ResultNode} so that a group can be labeled with + * whatever value the classifier expression returns. * * @param id The label to set. * @return This, to allow chaining. @@ -180,7 +174,7 @@ public class Group extends Identifiable { } /** - * <p>Sets the relevancy to use for this group.</p> + * Sets the relevancy to use for this group. * * @param rank The rank to set. * @return This, to allow chaining. @@ -190,17 +184,13 @@ public class Group extends Identifiable { return this; } - /** - * <p>Return the relevancy of this group.</p> - * - * @return Relevance. - */ + /** Return the rank score of this group. */ public double getRank() { return rank; } /** - * <p>Adds a child group to this.</p> + * Adds a child group to this. * * @param child The group to add. * @return This, to allow chaining. @@ -308,13 +298,7 @@ public class Group extends Identifiable { if (diff > 0) { return 1; } - if (rank > rhs.rank) { - return -1; - } - if (rank < rhs.rank) { - return 1; - } - return 0; + return -Double.compare(rank, rhs.rank); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Hit.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Hit.java index 6b2ce5c3b72..663bc18dc14 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Hit.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Hit.java @@ -79,14 +79,14 @@ public abstract class Hit extends Identifiable { return super.hashCode() + (int)rank; } - @SuppressWarnings({ "RedundantIfStatement", "EqualsWhichDoesntCheckParameterClass" }) + @SuppressWarnings({ "RedundantIfStatement" }) @Override public boolean equals(Object obj) { if (!super.equals(obj)) { return false; } Hit rhs = (Hit)obj; - if (rank != rhs.rank) { + if (Double.compare(rank, rhs.rank) != 0) { return false; } if (!equals(context, rhs.context)) { @@ -101,4 +101,5 @@ public abstract class Hit extends Identifiable { visitor.visit("rank", rank); visitor.visit("context", context); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java index 275f38f7350..63b2b881e01 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java @@ -73,8 +73,8 @@ public class HitsAggregationResult extends AggregationResult { /** * Sets the summary class of hits to collect. * - * @param summaryClass The summary class to collect. - * @return This, to allow chaining. + * @param summaryClass the summary class to collect. + * @return this, to allow chaining. */ public HitsAggregationResult setSummaryClass(String summaryClass) { this.summaryClass = summaryClass; @@ -84,8 +84,8 @@ public class HitsAggregationResult extends AggregationResult { /** * Sets the maximum number of hits to collect. * - * @param maxHits The number of hits to collect. - * @return This, to allow chaining. + * @param maxHits the number of hits to collect. + * @return this, to allow chaining. */ public HitsAggregationResult setMaxHits(int maxHits) { this.maxHits = maxHits; @@ -102,7 +102,7 @@ public class HitsAggregationResult extends AggregationResult { } /** - * Add a hit to this aggregation result + * Adds a hit to this aggregation result * * @param h the hit * @return this object @@ -159,11 +159,7 @@ public class HitsAggregationResult extends AggregationResult { @Override public void postMerge() { - Collections.sort(hits, new Comparator<Hit>() { - public int compare(Hit lhs, Hit rhs) { - return (lhs.getRank() > rhs.getRank()) ? -1 : (lhs.getRank() < rhs.getRank()) ? 1 : 0; - } - }); + hits.sort((lhs, rhs) -> -Double.compare(lhs.getRank(), rhs.getRank())); if ((maxHits >= 0) && (hits.size() > maxHits)) { hits = hits.subList(0, maxHits); } @@ -172,15 +168,9 @@ public class HitsAggregationResult extends AggregationResult { @Override protected boolean equalsAggregation(AggregationResult obj) { HitsAggregationResult rhs = (HitsAggregationResult)obj; - if (!summaryClass.equals(rhs.summaryClass)) { - return false; - } - if (maxHits != rhs.maxHits) { - return false; - } - if (!hits.equals(rhs.hits)) { - return false; - } + if ( ! summaryClass.equals(rhs.summaryClass)) return false; + if (maxHits != rhs.maxHits) return false; + if ( ! hits.equals(rhs.hits)) return false; return true; } @@ -215,4 +205,5 @@ public class HitsAggregationResult extends AggregationResult { hit.select(predicate, operation); } } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingTestCase.java index fe5405ecb6a..ec379e5f8af 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingTestCase.java @@ -223,5 +223,6 @@ public class GroupingTestCase { Grouping other = (Grouping)Grouping.create(buf); assertEquals(grouping, other); } + } diff --git a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp index 6ec379140c0..c24a11833e7 100644 --- a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp +++ b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp @@ -14,6 +14,13 @@ using namespace search::fef; using namespace search::queryeval; using search::attribute::ElementIterator; +void verify_elements(SameElementSearch &se, uint32_t docid, const std::initializer_list<uint32_t> list) { + std::vector<uint32_t> expect(list); + std::vector<uint32_t> actual; + se.find_matching_elements(docid, actual); + EXPECT_EQUAL(actual, expect); +} + std::unique_ptr<SameElementBlueprint> make_blueprint(const std::vector<FakeResult> &children, bool fake_attr = false) { auto result = std::make_unique<SameElementBlueprint>(false); for (size_t i = 0; i < children.size(); ++i) { @@ -61,6 +68,20 @@ TEST("require that simple match can be found") { EXPECT_EQUAL(result, expect); } +TEST("require that matching elements can be identified") { + auto a = make_result({{5, {1,3,7,12}}, {10, {1,2,3}}}); + auto b = make_result({{5, {3,5,7,10}}, {10, {4,5,6}}}); + auto bp = finalize(make_blueprint({a,b}), false); + auto md = MatchData::makeTestInstance(0, 0); + auto search = bp->createSearch(*md, false); + search->initRange(1, 1000); + SameElementSearch *se = dynamic_cast<SameElementSearch*>(search.get()); + ASSERT_TRUE(se != nullptr); + TEST_DO(verify_elements(*se, 5, {3, 7})); + TEST_DO(verify_elements(*se, 10, {})); + TEST_DO(verify_elements(*se, 20, {})); +} + TEST("require that children must match within same element") { auto a = make_result({{5, {1,3,7}}}); auto b = make_result({{5, {2,5,10}}}); diff --git a/searchlib/src/vespa/searchlib/common/featureset.h b/searchlib/src/vespa/searchlib/common/featureset.h index 1ec662685a7..beb9bd909ee 100644 --- a/searchlib/src/vespa/searchlib/common/featureset.h +++ b/searchlib/src/vespa/searchlib/common/featureset.h @@ -50,11 +50,8 @@ private: FeatureSet & operator=(const FeatureSet &); public: - /** - * Convenience typedef for a shared pointer to an object of this - * class. - **/ - typedef std::shared_ptr<FeatureSet> SP; + using SP = std::shared_ptr<FeatureSet>; + using UP = std::unique_ptr<FeatureSet>; /** * Create a new object without any feature information. diff --git a/searchlib/src/vespa/searchlib/common/matching_elements.h b/searchlib/src/vespa/searchlib/common/matching_elements.h index 9299191e83a..b31b258ea4c 100644 --- a/searchlib/src/vespa/searchlib/common/matching_elements.h +++ b/searchlib/src/vespa/searchlib/common/matching_elements.h @@ -24,6 +24,8 @@ public: MatchingElements(); ~MatchingElements(); + using UP = std::unique_ptr<MatchingElements>; + void add_matching_elements(uint32_t docid, const vespalib::string &struct_field_name, const std::vector<uint32_t> &elements); const std::vector<uint32_t> &get_matching_elements(uint32_t docid, const vespalib::string &struct_field_name) const; }; diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp index 8f3fc9c350d..37f8eccadd9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp @@ -114,4 +114,22 @@ SameElementSearch::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "strict", _strict); } +void +SameElementSearch::find_matching_elements(uint32_t docid, std::vector<uint32_t> &dst) +{ + if (check_docid_match(docid)) { + unpack_children(docid); + int32_t cand = 0; + while (cand >= 0) { + int32_t next = try_match(_childMatch, _iterators, cand); + if (next == cand) { + dst.push_back(cand); + ++cand; + } else { + cand = next; + } + } + } +} + } diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.h b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h index 1fd381eb1ae..ae7ecab781d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/common/matching_elements.h> #include <memory> #include <vector> @@ -40,6 +41,11 @@ public: void doUnpack(uint32_t) override {} void visitMembers(vespalib::ObjectVisitor &visitor) const override; const std::vector<SearchIterator::UP> &children() const { return _children; } + + // used during docsum fetching to identify matching elements + // initRange must be called before use. + // doSeek/doUnpack must not be called. + void find_matching_elements(uint32_t docid, std::vector<uint32_t> &dst); }; } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java index 3e392448ce2..78fad5f5867 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java @@ -9,7 +9,7 @@ import java.lang.reflect.InvocationTargetException; import java.util.HashMap; /** - * This is the base class to do cross-language serialization and deserialization of complete object structures without + * The base class to do cross-language serialization and deserialization of complete object structures without * the need for a separate protocol. Each subclass needs to register itself using the {@link #registerClass(int, Class)} * method, and override {@link #onGetClassId()} to return the same classId as the one registered. Creating an instance * of an identifiable object is done through the {@link #create(Deserializer)} or {@link #createFromId(int)} factory diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java index f1b7d6dc841..e60bb664d7d 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java @@ -2,17 +2,17 @@ package com.yahoo.vespa.objects; /** - * @author baldersheim - * - * This class acts as an interface for traversing a tree, or a graph. + * A node in a traversable tree. * Every non leaf Object implements {@link #selectMembers(ObjectPredicate, ObjectOperation)} implementing * the actual traversal. You can then implement an {@link ObjectPredicate} to select which nodes you want to look at with * your {@link ObjectOperation} + * + * @author baldersheim */ public class Selectable { /** - * Apply the predicate to this object. If the predicate returns true, pass this object to the operation, otherwise + * Applies the predicate to this object. If the predicate returns true, pass this object to the operation, otherwise * invoke the {@link #selectMembers(ObjectPredicate, ObjectOperation)} method to locate sub-elements that might * trigger the predicate. * @@ -28,7 +28,7 @@ public class Selectable { } /** - * Invoke {@link #select(ObjectPredicate, ObjectOperation)} on any member objects this object wants to expose + * Invokes {@link #select(ObjectPredicate, ObjectOperation)} on any member objects this object wants to expose * through the selection mechanism. Overriding this method is optional, and which objects to expose is determined by * the application logic of the object itself. * @@ -44,4 +44,5 @@ public class Selectable { selectable.select(predicate, operation); } } + } diff --git a/zkfacade/CMakeLists.txt b/zkfacade/CMakeLists.txt index 52eae58e108..6610356c0fd 100644 --- a/zkfacade/CMakeLists.txt +++ b/zkfacade/CMakeLists.txt @@ -1,8 +1,2 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_fat_java_artifact(zkfacade) -install_fat_java_artifact(zkctl) - -vespa_install_script(src/main/sh/vespa-zkcat vespa-zkcat bin) -vespa_install_script(src/main/sh/vespa-zkcli vespa-zkcli bin) -vespa_install_script(src/main/sh/vespa-zkctl vespa-zkctl bin) -vespa_install_script(src/main/sh/vespa-zkls vespa-zkls bin) diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index 2e016e71132..8b2a90abc12 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -107,30 +107,6 @@ </configuration> </plugin> <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <finalName>zkctl</finalName> - <archive> - <manifest> - <mainClass>org.apache.zookeeper.ZooKeeperMain</mainClass> - </manifest> - </archive> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <!-- append to the packaging phase. --> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> <groupId>com.yahoo.vespa</groupId> <artifactId>abi-check-plugin</artifactId> </plugin> diff --git a/zookeeper-command-line-client/CMakeLists.txt b/zookeeper-command-line-client/CMakeLists.txt index bbcffeff392..1dc4ade83b9 100644 --- a/zookeeper-command-line-client/CMakeLists.txt +++ b/zookeeper-command-line-client/CMakeLists.txt @@ -1,2 +1,7 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_fat_java_artifact(zookeeper-command-line-client) + +vespa_install_script(src/main/sh/vespa-zkcat vespa-zkcat bin) +vespa_install_script(src/main/sh/vespa-zkcli vespa-zkcli bin) +vespa_install_script(src/main/sh/vespa-zkctl vespa-zkctl bin) +vespa_install_script(src/main/sh/vespa-zkls vespa-zkls bin) diff --git a/zookeeper-command-line-client/src/main/resources/log4j-vespa.properties b/zookeeper-command-line-client/src/main/resources/log4j-vespa.properties new file mode 100644 index 00000000000..34a4b174335 --- /dev/null +++ b/zookeeper-command-line-client/src/main/resources/log4j-vespa.properties @@ -0,0 +1,6 @@ +log4j.rootLogger=WARN + +# CONSOLE is set to be a ConsoleAppender using a PatternLayout +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %m%n diff --git a/zkfacade/src/main/sh/vespa-zkcat b/zookeeper-command-line-client/src/main/sh/vespa-zkcat index e156fbb109e..e156fbb109e 100755 --- a/zkfacade/src/main/sh/vespa-zkcat +++ b/zookeeper-command-line-client/src/main/sh/vespa-zkcat diff --git a/zkfacade/src/main/sh/vespa-zkcli b/zookeeper-command-line-client/src/main/sh/vespa-zkcli index 33807d15346..5b23226dc5b 100755 --- a/zkfacade/src/main/sh/vespa-zkcli +++ b/zookeeper-command-line-client/src/main/sh/vespa-zkcli @@ -93,6 +93,6 @@ while [ $# -gt 0 ]; do done $sudo java \ - -cp $VESPA_HOME/lib/jars/zkctl-jar-with-dependencies.jar \ - -Dlog4j.configuration=file:$VESPA_HOME/etc/log4j-vespa.properties \ + -cp $VESPA_HOME/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar \ + -Dlog4j.configuration="log4j-vespa.properties" \ org.apache.zookeeper.ZooKeeperMain "$@" diff --git a/zkfacade/src/main/sh/vespa-zkctl b/zookeeper-command-line-client/src/main/sh/vespa-zkctl index 88165a1ed38..88165a1ed38 100755 --- a/zkfacade/src/main/sh/vespa-zkctl +++ b/zookeeper-command-line-client/src/main/sh/vespa-zkctl diff --git a/zkfacade/src/main/sh/vespa-zkls b/zookeeper-command-line-client/src/main/sh/vespa-zkls index c290d2ff3ac..c290d2ff3ac 100755 --- a/zkfacade/src/main/sh/vespa-zkls +++ b/zookeeper-command-line-client/src/main/sh/vespa-zkls |