diff options
135 files changed, 1740 insertions, 3341 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 6a8d754af1c..8ca0e5c501c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -258,7 +258,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override - public void validateXML() throws IOException { + public void validateXML() { if (failOnValidateXml) { throw new IllegalArgumentException("Error in application package"); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index da0566934f7..6ee525c1246 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -419,7 +419,8 @@ public final class ContainerCluster @NonNull public SearchChains getSearchChains() { if (containerSearch == null) - throw new IllegalStateException("Null search components!"); + throw new IllegalStateException("Search components not found in container cluster '" + getSubId() + + "': Add <search/> to the cluster in services.xml"); return containerSearch.getChains(); } @@ -479,7 +480,8 @@ public final class ContainerCluster @NonNull public DocprocChains getDocprocChains() { if (containerDocproc == null) - throw new IllegalStateException("Null docproc components!"); + throw new IllegalStateException("Document processing components not found in container cluster '" + getSubId() + + "': Add <document-processing/> to the cluster in services.xml"); return containerDocproc.getChains(); } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index d4ce445d50c..dd5a15acb2a 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1357,47 +1357,154 @@ public class ModelProvisioningTest { } @Test + public void testModelWithReferencedIndexingCluster() { + String services = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services version=\"1.0\">\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"vespa-1\"/>\n" + + " <configservers>\n" + + " <configserver hostalias=\"vespa-1\"/>\n" + + " </configservers>\n" + + " </admin>\n" + + "\n" + + " <container id=\"container\" version=\"1.0\">\n" + + " <document-processing/>\n" + + " <document-api/>\n" + + " <search/>\n" + + " <nodes jvmargs=\"-Xms512m -Xmx512m\">\n" + + " <node hostalias=\"vespa-1\"/>\n" + + " </nodes>\n" + + " </container>\n" + + "\n" + + " <content id=\"storage\" version=\"1.0\">\n" + + " <search>\n" + + " <visibility-delay>1.0</visibility-delay>\n" + + " </search>\n" + + " <redundancy>2</redundancy>\n" + + " <documents>\n" + + " <document type=\"type1\" mode=\"index\"/>\n" + + " <document-processing cluster=\"container\"/>\n" + + " </documents>\n" + + " <nodes>\n" + + " <node hostalias=\"vespa-1\" distribution-key=\"0\"/>\n" + + " </nodes>\n" + + " </content>\n" + + "\n" + + "</services>"; + + VespaModel model = createNonProvisionedMultitenantModel(services); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(1)); + ContentCluster content = model.getContentClusters().get("storage"); + assertEquals(1, content.getRootGroup().getNodes().size()); + ContainerCluster controller = content.getClusterControllers(); + assertEquals(1, controller.getContainers().size()); + } + + @Test + public void testSharedNodesNotHosted() { + String hosts = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<hosts>\n" + + " <host name=\"vespa-1\">\n" + + " <alias>vespa-1</alias>\n" + + " </host>\n" + + " <host name=\"vespa-2\">\n" + + " <alias>vespa-2</alias>\n" + + " </host>\n" + + " <host name=\"vespa-3\">\n" + + " <alias>vespa-3</alias>\n" + + " </host>\n" + + "</hosts>"; + String services = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<services version=\"1.0\">\n" + + "\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"vespa-1\"/>\n" + + " <configservers>\n" + + " <configserver hostalias=\"vespa-1\"/>\n" + + " </configservers>\n" + + " </admin>\n" + + "\n" + + " <container id=\"container\" version=\"1.0\">\n" + + " <document-processing/>\n" + + " <document-api/>\n" + + " <search/>\n" + + " <nodes jvmargs=\"-Xms512m -Xmx512m\">\n" + + " <node hostalias=\"vespa-1\"/>\n" + + " <node hostalias=\"vespa-2\"/>\n" + + " <node hostalias=\"vespa-3\"/>\n" + + " </nodes>\n" + + " </container>\n" + + "\n" + + " <content id=\"storage\" version=\"1.0\">\n" + + " <search>\n" + + " <visibility-delay>1.0</visibility-delay>\n" + + " </search>\n" + + " <redundancy>2</redundancy>\n" + + " <documents>\n" + + " <document type=\"type1\" mode=\"index\"/>\n" + + " <document-processing cluster=\"container\"/>\n" + + " </documents>\n" + + " <nodes>\n" + + " <node hostalias=\"vespa-1\" distribution-key=\"0\"/>\n" + + " <node hostalias=\"vespa-2\" distribution-key=\"1\"/>\n" + + " <node hostalias=\"vespa-3\" distribution-key=\"2\"/>\n" + + " </nodes>\n" + + " </content>\n" + + "\n" + + "</services>"; + + VespaModel model = createNonProvisionedModel(false, hosts, services); + assertEquals(3, model.getRoot().getHostSystem().getHosts().size()); + ContentCluster content = model.getContentClusters().get("storage"); + assertEquals(3, content.getRootGroup().getNodes().size()); + } + + @Test public void testMultitenantButNotHostedSharedContentNode() { String services = - "<?xml version='1.0' encoding='UTF-8' ?>" + - "<services version='1.0'>" + - " <admin version='2.0'>" + - " <adminserver hostalias='node1'/>" + - " </admin>" + - " <jdisc id='default' version='1.0'>" + - " <search/>" + - " <nodes>" + - " <node hostalias='node1'/>" + - " </nodes>" + - " </jdisc>" + - " <content id='storage' version='1.0'>" + - " <redundancy>2</redundancy>" + - " <group>" + - " <node distribution-key='0' hostalias='node1'/>" + - " <node distribution-key='1' hostalias='node1'/>" + - " </group>" + - " <tuning>" + - " <cluster-controller>" + - " <transition-time>0</transition-time>" + - " </cluster-controller>" + - " </tuning>" + - " <documents>" + - " <document mode='store-only' type='type1'/>" + - " </documents>" + - " <engine>" + - " <proton/>" + - " </engine>" + - " </content>" + - " <content id='search' version='1.0'>" + - " <redundancy>2</redundancy>" + - " <group>" + - " <node distribution-key='0' hostalias='node1'/>" + - " </group>" + - " <documents>" + - " <document type='type1'/>" + - " </documents>" + - " </content>" + - " </services>"; + "<?xml version='1.0' encoding='UTF-8' ?>" + + "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node1'/>" + + " </admin>" + + " <jdisc id='default' version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node1'/>" + + " </nodes>" + + " </jdisc>" + + " <content id='storage' version='1.0'>" + + " <redundancy>2</redundancy>" + + " <group>" + + " <node distribution-key='0' hostalias='node1'/>" + + " <node distribution-key='1' hostalias='node1'/>" + + " </group>" + + " <tuning>" + + " <cluster-controller>" + + " <transition-time>0</transition-time>" + + " </cluster-controller>" + + " </tuning>" + + " <documents>" + + " <document mode='store-only' type='type1'/>" + + " </documents>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " </content>" + + " <content id='search' version='1.0'>" + + " <redundancy>2</redundancy>" + + " <group>" + + " <node distribution-key='0' hostalias='node1'/>" + + " </group>" + + " <documents>" + + " <document type='type1'/>" + + " </documents>" + + " </content>" + + " </services>"; VespaModel model = createNonProvisionedMultitenantModel(services); assertThat(model.getRoot().getHostSystem().getHosts().size(), is(1)); @@ -1408,10 +1515,14 @@ public class ModelProvisioningTest { } private VespaModel createNonProvisionedMultitenantModel(String services) { - VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); + return createNonProvisionedModel(true, null, services); + } + + private VespaModel createNonProvisionedModel(boolean multitenant, String hosts, String services) { + VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSearchDefinition("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; DeployState deployState = new DeployState.Builder().applicationPackage(appPkg). - properties((new DeployProperties.Builder()).multitenant(true).build()). + properties((new DeployProperties.Builder()).multitenant(multitenant).build()). build(true); return modelCreatorWithMockPkg.create(false, deployState); } diff --git a/config-provisioning/src/main/resources/configdefinitions/node-repository.def b/config-provisioning/src/main/resources/configdefinitions/node-repository.def index 8ea9265aa23..4af8806fbbb 100644 --- a/config-provisioning/src/main/resources/configdefinitions/node-repository.def +++ b/config-provisioning/src/main/resources/configdefinitions/node-repository.def @@ -6,3 +6,6 @@ namespace=config.provisioning dockerImage string default="dummyImage" useCuratorClientCache bool default=false + +# Comma separated list of hostnames that are authorized for all config server REST APIs +hostnameWhitelist string default="" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3cef4b9de61..ad52de5eede 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.FileDistributionStatus; import com.yahoo.vespa.config.server.application.HttpProxy; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.RefeedActions; @@ -80,7 +79,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Tenants tenants; private final Optional<Provisioner> hostProvisioner; - private final LogServerLogGrabber logServerLogGrabber; private final ApplicationConvergenceChecker convergeChecker; private final HttpProxy httpProxy; private final Clock clock; @@ -92,11 +90,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye @Inject public ApplicationRepository(Tenants tenants, HostProvisionerProvider hostProvisionerProvider, - LogServerLogGrabber logServerLogGrabber, ApplicationConvergenceChecker applicationConvergenceChecker, HttpProxy httpProxy, ConfigserverConfig configserverConfig) { - this(tenants, hostProvisionerProvider.getHostProvisioner(), logServerLogGrabber, + this(tenants, hostProvisionerProvider.getHostProvisioner(), applicationConvergenceChecker, httpProxy, configserverConfig, Clock.systemUTC(), new FileDistributionStatus()); } @@ -104,14 +101,13 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public ApplicationRepository(Tenants tenants, Provisioner hostProvisioner, Clock clock) { - this(tenants, Optional.of(hostProvisioner), new LogServerLogGrabber(), + this(tenants, Optional.of(hostProvisioner), new ApplicationConvergenceChecker(), new HttpProxy(new SimpleHttpFetcher()), new ConfigserverConfig(new ConfigserverConfig.Builder()), clock, new FileDistributionStatus()); } private ApplicationRepository(Tenants tenants, Optional<Provisioner> hostProvisioner, - LogServerLogGrabber logServerLogGrabber, ApplicationConvergenceChecker applicationConvergenceChecker, HttpProxy httpProxy, ConfigserverConfig configserverConfig, @@ -119,7 +115,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye FileDistributionStatus fileDistributionStatus) { this.tenants = tenants; this.hostProvisioner = hostProvisioner; - this.logServerLogGrabber = logServerLogGrabber; this.convergeChecker = applicationConvergenceChecker; this.httpProxy = httpProxy; this.clock = clock; @@ -217,11 +212,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return true; } - public String grabLog(Tenant tenant, ApplicationId applicationId) { - Application application = getApplication(tenant, applicationId); - return logServerLogGrabber.grabLog(application); - } - public HttpResponse serviceConvergenceCheck(Tenant tenant, ApplicationId applicationId, String hostname, URI uri) { Application application = getApplication(tenant, applicationId); return convergeChecker.serviceConvergenceCheck(application, hostname, uri); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java deleted file mode 100644 index b29953187e5..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.application; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.model.api.PortInfo; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.config.server.http.InternalServerException; -import com.yahoo.yolean.Exceptions; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.Socket; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * Fetches log entries from logserver with level errors and fatal. The logserver only returns - * a log entry once over this API so doing repeated calls will not give the same results. - * - * @author dybis - */ -public class LogServerLogGrabber extends AbstractComponent { - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(LogServerLogGrabber.class.getName()); - - public LogServerLogGrabber() {} - - public String grabLog(Application application) { - LogServerInfo logServerConnectionInfo = findLogserverConnectionInfo(application); - log.log(LogLevel.DEBUG, "Requested error logs, pulling from logserver on " + logServerConnectionInfo); - try { - return readLog(logServerConnectionInfo.hostName, logServerConnectionInfo.port); - } catch (IOException e) { - throw new InternalServerException(Exceptions.toMessageString(e)); - } - } - - private LogServerInfo findLogserverConnectionInfo(Application application) { - List<LogServerInfo> logServerConnectionInfos = new ArrayList<>(); - application.getModel().getHosts() - .forEach(host -> host.getServices().stream() - .filter(service -> service.getServiceType().equals("logserver")) - .forEach(logService -> { - Optional<Integer> logPort = getErrorLogPort(logService); - logPort.ifPresent(port -> logServerConnectionInfos.add(new LogServerInfo(host.getHostname(), port))); - })); - - if (logServerConnectionInfos.size() > 1) throw new RuntimeException("Found several log server ports"); - if (logServerConnectionInfos.size() == 0) throw new InternalServerException("Did not find any log server in config model"); - - return logServerConnectionInfos.get(0); - } - - // Protected to be able to test - protected String readLog(String host, int port) throws IOException { - Socket socket = new Socket(host, port); - BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream())); - StringBuilder data = new StringBuilder(); - - int bufferSize = 4096; - int charsRead; - do { - char[] buffer = new char[bufferSize]; - charsRead = in.read(buffer); - data.append(new String(buffer, 0, charsRead)); - } while (charsRead == bufferSize); - in.close(); - socket.close(); - return data.toString(); - } - - private Optional<Integer> getErrorLogPort(ServiceInfo service) { - return service.getPorts().stream() - .filter(port -> port.getTags().contains("last-errors-holder")) - .map(PortInfo::getPort) - .findFirst(); - } - - private class LogServerInfo { - String hostName; - int port; - - LogServerInfo(String hostName, int port) { - this.hostName = hostName; - this.port = port; - } - - public String toString() { - return hostName + ":" + port; - } - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 7be7bfb47e1..42fdb16c7ca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -16,16 +16,12 @@ import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.ContentRequest; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; import com.yahoo.vespa.config.server.tenant.Tenant; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.time.Duration; /** @@ -104,12 +100,9 @@ public class ApplicationHandler extends HttpHandler { @Override public HttpResponse handlePOST(HttpRequest request) { ApplicationId applicationId = getApplicationIdFromRequest(request); - Tenant tenant = verifyTenantAndApplication(applicationId); if (request.getUri().getPath().endsWith("restart")) return restart(request, applicationId); - if (request.getUri().getPath().endsWith("log")) - return grabLog(request, applicationId, tenant); - throw new NotFoundException("Illegal POST request '" + request.getUri() + "': Must end by /restart or /log"); + throw new NotFoundException("Illegal POST request '" + request.getUri() + "': Must end with /restart"); } private HttpResponse restart(HttpRequest request, ApplicationId applicationId) { @@ -120,24 +113,6 @@ public class ApplicationHandler extends HttpHandler { return new JSONResponse(Response.Status.OK); // return empty } - private HttpResponse grabLog(HttpRequest request, ApplicationId applicationId, Tenant tenant) { - if (getBindingMatch(request).groupCount() != 7) - throw new NotFoundException("Illegal POST log request '" + request.getUri() + - "': Must have 6 arguments but had " + ( getBindingMatch(request).groupCount()-1 ) ); - final String response = applicationRepository.grabLog(tenant, applicationId); - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write(response.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; - } - }; - } - private HostFilter hostFilterFrom(HttpRequest request) { return HostFilter.from(request.getProperty("hostname"), request.getProperty("flavor"), @@ -157,7 +132,6 @@ public class ApplicationHandler extends HttpHandler { return HttpConfigRequests.getBindingMatch(request, // WARNING: UPDATE src/main/resources/configserver-app/services.xml IF YOU MAKE ANY CHANGES TO THESE BINDINGS! "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/log", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge", diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index f5b503e54d8..9419a09ce7e 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -129,8 +129,6 @@ <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> - <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/log</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/log</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge</binding> <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge</binding> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index ef53baf821d..9a69dfc5f07 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.config.server.SuperModelGenerationCounter; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; import com.yahoo.vespa.config.server.application.HttpProxy; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.StaticResponse; @@ -90,8 +89,7 @@ public class ApplicationHandlerTest { mockHandler = createMockApplicationHandler( provisioner, new ApplicationConvergenceChecker(stateApiFactory), - mockHttpProxy, - new MockLogServerLogGrabber()); + mockHttpProxy); listApplicationsHandler = new ListApplicationsHandler( ListApplicationsHandler.testOnlyContext(), tenants, Zone.defaultZone()); @@ -100,14 +98,12 @@ public class ApplicationHandlerTest { private ApplicationHandler createMockApplicationHandler( Provisioner provisioner, ApplicationConvergenceChecker convergeChecker, - HttpProxy httpProxy, - LogServerLogGrabber logServerLogGrabber) { + HttpProxy httpProxy) { return new ApplicationHandler( ApplicationHandler.testOnlyContext(), Zone.defaultZone(), new ApplicationRepository(tenants, HostProvisionerProvider.withProvisioner(provisioner), - logServerLogGrabber, convergeChecker, httpProxy, new ConfigserverConfig(new ConfigserverConfig.Builder()))); @@ -119,7 +115,6 @@ public class ApplicationHandlerTest { Zone.defaultZone(), new ApplicationRepository(tenants, HostProvisionerProvider.withProvisioner(provisioner), - new LogServerLogGrabber(), new ApplicationConvergenceChecker(stateApiFactory), new HttpProxy(new SimpleHttpFetcher()), new ConfigserverConfig(new ConfigserverConfig.Builder()))); @@ -215,14 +210,6 @@ public class ApplicationHandlerTest { } @Test - public void testGrabLog() throws Exception { - long sessionId = 1; - ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - addMockApplication(tenants.getTenant(mytenantName), application, sessionId, Clock.systemUTC()); - assertEquals("log line", grabLog(application, Zone.defaultZone())); - } - - @Test public void testClusterControllerStatus() throws Exception { long sessionId = 1; ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); @@ -249,8 +236,7 @@ public class ApplicationHandlerTest { mockHandler = createMockApplicationHandler( provisioner, new ApplicationConvergenceChecker(stateApiFactory), - new HttpProxy(new SimpleHttpFetcher()), - new LogServerLogGrabber()); + new HttpProxy(new SimpleHttpFetcher())); final ApplicationId applicationId = ApplicationId.defaultId(); addMockApplication(tenants.getTenant(mytenantName), applicationId, 1, Clock.systemUTC()); assertApplicationExists(mytenantName, applicationId, Zone.defaultZone()); @@ -405,13 +391,6 @@ public class ApplicationHandlerTest { HandlerTest.assertHttpStatusCodeAndMessage(response, 200, ""); } - private String grabLog(ApplicationId application, Zone zone) throws IOException { - String restartUrl = toUrlPath(application, zone, true) + "/log"; - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.POST)); - HandlerTest.assertHttpStatusCodeAndMessage(response, 200, ""); - return SessionHandlerTest.getRenderedString(response); - } - private HttpResponse fileDistributionStatus(ApplicationId application, Zone zone) { String restartUrl = toUrlPath(application, zone, true) + "/filedistributionstatus"; return mockHandler.handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); @@ -432,10 +411,4 @@ public class ApplicationHandlerTest { } } - private static class MockLogServerLogGrabber extends LogServerLogGrabber { - @Override - protected String readLog(String host, int port) throws IOException { - return "log line"; - } - } } diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java index abbe346e6e9..52d26236ec7 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -266,13 +266,12 @@ public class LogFileHandler extends StreamHandler { } } - private void triggerCompression(String oldFileName) throws InterruptedException { + private void triggerCompression(String oldFileName) { try { Runtime r = Runtime.getRuntime(); Process p = r.exec(new String[] { "gzip", oldFileName }); // Detonator pattern: Think of all the fun we can have if gzip isn't what we // think it is, if it doesn't return, etc, etc - p.waitFor(); } catch (IOException e) { // little we can do... } diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 68736c39623..4b824edbb21 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -439,7 +439,7 @@ <findbugs.version>1.3.9</findbugs.version> <guava.version>18.0</guava.version> <guice.version>3.0</guice.version> - <jetty.version>9.4.8.v20171121</jetty.version> + <jetty.version>9.4.9.v20180320</jetty.version> <scala.version>2.11.4</scala.version> <!-- When updating this, the scala.major-version in parent must also be updated! --> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index 3d026172c86..22b049c9ab7 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -4,6 +4,7 @@ package com.yahoo.container.jdisc.metric; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -34,13 +35,13 @@ public class MetricUpdater extends AbstractComponent { private final Scheduler scheduler; @Inject - public MetricUpdater(Metric metric) { - this(new TimerScheduler(), metric); + public MetricUpdater(Metric metric, ContainerWatchdogMetrics containerWatchdogMetrics) { + this(new TimerScheduler(), metric, containerWatchdogMetrics); } - MetricUpdater(Scheduler scheduler, Metric metric) { + MetricUpdater(Scheduler scheduler, Metric metric, ContainerWatchdogMetrics containerWatchdogMetrics) { this.scheduler = scheduler; - scheduler.schedule(new UpdaterTask(metric), Duration.ofSeconds(10)); + scheduler.schedule(new UpdaterTask(metric, containerWatchdogMetrics), Duration.ofSeconds(10)); } @Override @@ -87,9 +88,11 @@ public class MetricUpdater extends AbstractComponent { private final Runtime runtime = Runtime.getRuntime(); private final Metric metric; + private final ContainerWatchdogMetrics containerWatchdogMetrics; - public UpdaterTask(Metric metric) { + public UpdaterTask(Metric metric, ContainerWatchdogMetrics containerWatchdogMetrics) { this.metric = metric; + this.containerWatchdogMetrics = containerWatchdogMetrics; } @SuppressWarnings("deprecation") @@ -106,6 +109,7 @@ public class MetricUpdater extends AbstractComponent { metric.set(TOTAL_MEMORY_BYTES, totalMemory, null); metric.set(MEMORY_MAPPINGS_COUNT, count_mappings(), null); metric.set(OPEN_FILE_DESCRIPTORS, count_open_files(), null); + containerWatchdogMetrics.emitMetrics(metric); } } diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java index d94cea033f5..f10af7593a4 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java @@ -2,6 +2,7 @@ package com.yahoo.container.jdisc.metric; import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; import org.junit.Test; import java.time.Duration; @@ -20,7 +21,9 @@ public class MetricUpdaterTest { @Test public void metrics_are_updated_in_scheduler_cycle() throws InterruptedException { Metric metric = mock(Metric.class); - new MetricUpdater(new MockScheduler(), metric); + ContainerWatchdogMetrics containerWatchdogMetrics = mock(ContainerWatchdogMetrics.class); + new MetricUpdater(new MockScheduler(), metric, containerWatchdogMetrics); + verify(containerWatchdogMetrics, times(1)).emitMetrics(any()); verify(metric, times(8)).set(anyString(), any(), any()); } diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index 0709baa9be2..b8242ff5101 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.objects.BufferSerializer; import java.nio.ByteBuffer; import java.util.List; - /** * An "extended query" packet. This is the query packets used today, * they allow more flexible sets of parameters to be shipped with queries. @@ -26,12 +25,13 @@ import java.util.List; */ public class QueryPacket extends Packet { - final private Query query; + private final Query query; + private QueryPacketData queryPacketData; - int sessionOffset = 0; // Start of sessionKey ignore section for cache key - int sessionSize = 0; // Length of sessionKey ignore section for cache key - int ignoreableOffset = 0; // Start of (hits/offset/timestamp) ignore section for cache key - int ignoreableSize = 0; // Length of (hits/offset/timestamp) ignore section for cache key + private int sessionOffset = 0; // Start of sessionKey ignore section for cache key + private int sessionSize = 0; // Length of sessionKey ignore section for cache key + private int ignoreableOffset = 0; // Start of (hits/offset/timestamp) ignore section for cache key + private int ignoreableSize = 0; // Length of (hits/offset/timestamp) ignore section for cache key private QueryPacket(Query query) { this.query = query; @@ -80,6 +80,7 @@ public class QueryPacket extends Packet { private int getSessionKeySkipLength() { return (sessionSize > 0) ? sessionSize + 4 : 0; } + /** * Returns an opaque cache key for the query represented by this * (pre-serialized) packet. @@ -191,14 +192,14 @@ public class QueryPacket extends Packet { /** * feature bits, taken from searchlib/common/transport.h - **/ - static final int QF_PARSEDQUERY = 0x00000002; - static final int QF_RANKP = 0x00000004; - static final int QF_SORTSPEC = 0x00000080; - static final int QF_LOCATION = 0x00000800; - static final int QF_PROPERTIES = 0x00100000; - static final int QF_GROUPSPEC = 0x00400000; - static final int QF_SESSIONID = 0x00800000; + */ + private static final int QF_PARSEDQUERY = 0x00000002; + private static final int QF_RANKP = 0x00000004; + private static final int QF_SORTSPEC = 0x00000080; + private static final int QF_LOCATION = 0x00000800; + private static final int QF_PROPERTIES = 0x00100000; + private static final int QF_GROUPSPEC = 0x00400000; + private static final int QF_SESSIONID = 0x00800000; private int getFeatureInt(boolean sendSessionId) { int features = QF_PARSEDQUERY | QF_RANKP; // this bitmask means "parsed query" in query packet. @@ -215,27 +216,27 @@ public class QueryPacket extends Packet { /** * query flag bits, taken from searchlib/common/transport.h - **/ - static final int QFLAG_EXTENDED_COVERAGE = 0x00000001; - static final int QFLAG_COVERAGE_NODES = 0x00000002; - static final int QFLAG_ESTIMATE = 0x00000080; - static final int QFLAG_DROP_SORTDATA = 0x00004000; - static final int QFLAG_NO_RESULTCACHE = 0x00010000; - static final int QFLAG_DUMP_FEATURES = 0x00040000; + */ + private static final int QFLAG_EXTENDED_COVERAGE = 0x00000001; + private static final int QFLAG_COVERAGE_NODES = 0x00000002; + private static final int QFLAG_ESTIMATE = 0x00000080; + private static final int QFLAG_DROP_SORTDATA = 0x00004000; + private static final int QFLAG_NO_RESULTCACHE = 0x00010000; + private static final int QFLAG_DUMP_FEATURES = 0x00040000; private int getFlagInt() { int flags = getQueryFlags(query); queryPacketData.setQueryFlags(flags); - /** + /* * QFLAG_DROP_SORTDATA * SORTDATA is a mangling of data from the attribute vectors * which were used in the search which is byte comparable in * such a way the comparing SORTDATA for two different hits * will reproduce the order in which the data were returned when - * using sortspec. For now we simple drop these, but if they - * should be necessary at later date, QueryResultPacket must be - * updated to be able to parse result packets correctly. + * using sortspec. For now we simply drop these. If they + * become necessar, QueryResultPacket must be + * updated to be able to read the sort data. */ flags |= QFLAG_DROP_SORTDATA; return flags; @@ -264,13 +265,12 @@ public class QueryPacket extends Packet { * creating a summary request. * * @return wrapper object suitable for creating a summary fetch packet - * @throws IllegalStateException - * if no wrapper has been generated + * @throws IllegalStateException if no wrapper has been generated */ public QueryPacketData getQueryPacketData() { - if (queryPacketData == null) { + if (queryPacketData == null) throw new IllegalStateException("Trying to fetch a hit tag without having encoded the packet first."); - } return queryPacketData; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 2709755da5d..258f0dec9ff 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -13,6 +13,7 @@ import com.yahoo.container.handler.VipStatus; import com.yahoo.fs4.mplex.Backend; import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.net.HostName; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.IndexFacts; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java index c1c818848ad..b7e16b6c082 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java @@ -12,7 +12,7 @@ import com.yahoo.processing.request.CompoundName; /** * The cache control logic for FastSearcher * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class CacheControl { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 4fd0f884903..d50006fb82c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -21,20 +21,33 @@ import java.util.Map; */ public class DocsumDefinition { - private String name; - private final List<DocsumField> fields; + private final String name; + private final ImmutableList<DocsumField> fields; /** True if this contains dynamic fields */ - private boolean dynamic = false; + private final boolean dynamic; // Mapping between field names and their index in this.fields - private final Map<String,Integer> fieldNameToIndex; + private final ImmutableMap<String, Integer> fieldNameToIndex; + + public DocsumDefinition(String name, List<DocsumField> fields) { + this.name = name; + this.dynamic = false; + this.fields = ImmutableList.copyOf(fields); + ImmutableMap.Builder<String, Integer> fieldNameToIndexBuilder = new ImmutableMap.Builder<>(); + int i = 0; + for (DocsumField field : fields) + fieldNameToIndexBuilder.put(field.name, i++); + this.fieldNameToIndex = fieldNameToIndexBuilder.build(); + } + // TODO: Remove LegacyEmulationConfig (the config, not just the usage) on Vespa 7 DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config, LegacyEmulationConfig emulConfig) { this.name = config.name(); - List<DocsumField> fieldsBuilder = new ArrayList<>(); - Map<String,Integer> fieldNameToIndexBuilder = new HashMap<>(); + List<DocsumField> fieldsBuilder = new ArrayList<>(); + Map<String, Integer> fieldNameToIndexBuilder = new HashMap<>(); + boolean dynamic = false; for (DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field : config.fields()) { // no, don't switch the order of the two next lines :) fieldNameToIndexBuilder.put(field.name(), fieldsBuilder.size()); @@ -42,6 +55,7 @@ public class DocsumDefinition { if (field.dynamic()) dynamic = true; } + this.dynamic = dynamic; fields = ImmutableList.copyOf(fieldsBuilder); fieldNameToIndex = ImmutableMap.copyOf(fieldNameToIndexBuilder); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index 8d882adeb02..3ecc01a5e8e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; +import com.google.common.collect.ImmutableMap; import com.yahoo.slime.BinaryFormat; import com.yahoo.data.access.Inspector; import com.yahoo.slime.Slime; @@ -10,9 +11,12 @@ import com.yahoo.container.search.LegacyEmulationConfig; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.data.access.Type.OBJECT; @@ -27,19 +31,28 @@ public final class DocsumDefinitionSet { public static final int SLIME_MAGIC_ID = 0x55555555; private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); - private final HashMap<String, DocsumDefinition> definitionsByName = new HashMap<>(); + private final Map<String, DocsumDefinition> definitionsByName; private final LegacyEmulationConfig emulationConfig; public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config) { - this.emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder()); - configure(config); + this(config, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); } public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + this(toDocsums(config, emulConfig), emulConfig); + } + + public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions) { + this(docsumDefinitions, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); + } + + public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions, LegacyEmulationConfig emulConfig) { + this.definitionsByName = ImmutableMap.copyOf(docsumDefinitions.stream().collect(Collectors.toMap(DocsumDefinition::getName, p -> p))); this.emulationConfig = emulConfig; - configure(config); } + LegacyEmulationConfig legacyEmulationConfig() { return emulationConfig; } + /** * Returns a docsum definition by name, or null if not found * @@ -106,14 +119,13 @@ public final class DocsumDefinitionSet { return definitionsByName.size(); } - private void configure(DocumentdbInfoConfig.Documentdb config) { - for (int i = 0; i < config.summaryclass().size(); ++i) { - DocumentdbInfoConfig.Documentdb.Summaryclass sc = config.summaryclass(i); - DocsumDefinition docSumDef = new DocsumDefinition(sc, emulationConfig); - definitionsByName.put(sc.name(), docSumDef); - } - if (definitionsByName.size() == 0) { + private static Collection<DocsumDefinition> toDocsums(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + Collection<DocsumDefinition> docsums = new ArrayList<>(); + for (int i = 0; i < config.summaryclass().size(); ++i) + docsums.add(new DocsumDefinition(config.summaryclass(i), emulConfig)); + if (docsums.isEmpty()) log.warning("No summary classes found in DocumentdbInfoConfig.Documentdb"); - } + return docsums; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 1e44a8fa64d..d5e4eb75931 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -73,8 +73,8 @@ public abstract class DocsumField { this.name = name; } - /* for unit test only */ - static DocsumField create(String name, String typename) { + /* For unit test only */ + public static DocsumField create(String name, String typename) { return create(name, typename, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); } @@ -114,7 +114,7 @@ public abstract class DocsumField { /** * Convert a generic value into an object of the appropriate type * for this field. - **/ + */ public abstract Object convert(Inspector value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index 3366f92384a..0ae0983a1ae 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -4,10 +4,13 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableMap; import com.yahoo.container.search.LegacyEmulationConfig; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Representation of a back-end document database. @@ -24,12 +27,16 @@ public class DocumentDatabase { private final String name; private final DocsumDefinitionSet docsumDefSet; - private final Map<String, RankProfile> rankProfiles; + private final ImmutableMap<String, RankProfile> rankProfiles; public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb, LegacyEmulationConfig emulConfig) { - this.name = documentDb.name(); - this.docsumDefSet = new DocsumDefinitionSet(documentDb, emulConfig); - this.rankProfiles = ImmutableMap.copyOf(toRankProfiles(documentDb.rankprofile())); + this(documentDb.name(), new DocsumDefinitionSet(documentDb, emulConfig), toRankProfiles(documentDb.rankprofile())); + } + + public DocumentDatabase(String name, DocsumDefinitionSet docsumDefinitionSet, Collection<RankProfile> rankProfiles) { + this.name = name; + this.docsumDefSet = docsumDefinitionSet; + this.rankProfiles = ImmutableMap.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); } public String getName() { @@ -43,11 +50,14 @@ public class DocumentDatabase { /** Returns an unmodifiable map of all the rank profiles in this indexed by rank profile name */ public Map<String, RankProfile> rankProfiles() { return rankProfiles; } - private Map<String, RankProfile> toRankProfiles(List<DocumentdbInfoConfig.Documentdb.Rankprofile> rankProfileConfigList) { - Map<String, RankProfile> rankProfiles = new HashMap<>(); - for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) { - rankProfiles.put(c.name(), new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); - } + private static ImmutableMap<String, RankProfile> toMap(Collection<RankProfile> rankProfiles) { + return ImmutableMap.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); + } + + private static Collection<RankProfile> toRankProfiles(Collection<DocumentdbInfoConfig.Documentdb.Rankprofile> rankProfileConfigList) { + List<RankProfile> rankProfiles = new ArrayList<>(); + for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) + rankProfiles.add(new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures())); return rankProfiles; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index aab8aae6025..b3eaee8698a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -74,9 +74,10 @@ public class FastHit extends Hit { setPartId(0, 0); } + @Override public String toString() { return super.toString() + " [fasthit, globalid: " + globalId + ", partId: " - + partId + ", distributionkey: " + distributionKey + "]"; + + partId + ", distributionkey: " + distributionKey + "]"; } public static String asHexString(GlobalId gid) { @@ -267,7 +268,8 @@ public class FastHit extends Hit { this.distributionKey = distributionKey; } - void addSummary(DocsumDefinition docsumDef, Inspector value) { + /** For internal use */ + public void addSummary(DocsumDefinition docsumDef, Inspector value) { reserve(docsumDef.getFieldCount()); for (DocsumField field : docsumDef.getFields()) { String fieldName = field.getName(); @@ -290,10 +292,8 @@ public class FastHit extends Hit { * easier. This is not a method to be used for efficiency, as it causes * object allocations. * - * @param fieldName - * the name of the field to insert undecoded UTF-8 into - * @param value - * an array of valid UTF-8 data + * @param fieldName the name of the field to insert undecoded UTF-8 into + * @param value an array of valid UTF-8 data */ @Beta public void setLazyStringField(String fieldName, byte[] value) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 7d822fd603b..e5f255f18c0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.fastsearch; import java.util.Optional; import com.yahoo.compress.CompressionType; +import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.fs4.BasicPacket; import com.yahoo.fs4.ChannelTimeoutException; import com.yahoo.fs4.GetDocSumsPacket; @@ -15,7 +16,6 @@ import com.yahoo.fs4.QueryResultPacket; import com.yahoo.fs4.mplex.Backend; import com.yahoo.fs4.mplex.FS4Channel; import com.yahoo.fs4.mplex.InvalidChannelException; -import com.yahoo.net.HostName; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.prelude.querytransform.QueryRewrite; @@ -56,6 +56,9 @@ public class FastSearcher extends VespaBackEndSearcher { /** If this is turned on this will make search queries directly to the local search node when possible */ private final static CompoundName dispatchDirect = new CompoundName("dispatch.direct"); + /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */ + private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); + /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ private final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -231,6 +234,7 @@ public class FastSearcher extends VespaBackEndSearcher { /** * Only used to fill the sddocname field when using direct dispatching as that is normally done in VespaBackEndSearcher.decodeSummary + * * @param result The result */ private void fillSDDocName(Result result) { @@ -255,11 +259,15 @@ public class FastSearcher extends VespaBackEndSearcher { Query query = result.getQuery(); traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass)); - if (wantsRPCSummaryFill(query)) { + if (query.properties().getBoolean(dispatchSummaries, true) + && ! summaryNeedsQuery(query) + && ! cacheControl.useCache(query) + && ! legacyEmulationConfigIsSet(getDocumentDatabase(query))) { + CompressionType compression = CompressionType.valueOf(query.properties().getString(dispatchCompression, "LZ4").toUpperCase()); fillSDDocName(result); - dispatcher.fill(result, summaryClass, compression); + dispatcher.fill(result, summaryClass, getDocumentDatabase(query), compression); return; } @@ -349,6 +357,14 @@ public class FastSearcher extends VespaBackEndSearcher { } } + private boolean legacyEmulationConfigIsSet(DocumentDatabase db) { + LegacyEmulationConfig config = db.getDocsumDefinitionSet().legacyEmulationConfig(); + if (config.forceFillEmptyFields()) return true; + if (config.stringBackedFeatureData()) return true; + if (config.stringBackedStructuredData()) return true; + return false; + } + private static @NonNull Optional<String> quotedSummaryClass(String summaryClass) { return Optional.of(summaryClass == null ? "[null]" : quote(summaryClass)); } @@ -465,14 +481,9 @@ public class FastSearcher extends VespaBackEndSearcher { } boolean couldSend = channel.sendPacket(docsumsPacket); - if (isLoggingFine()) - getLogger().finest("Sent " + docsumsPacket + " on " + channel); if ( ! couldSend) throw new IOException("Could not successfully send GetDocSumsPacket."); receivedPackets = channel.receivePackets(result.getQuery().getTimeLeft(), docsumsPacket.getNumDocsums() + 1); - if (isLoggingFine()) - getLogger().finest("got " + receivedPackets.length + "docsumPackets"); - return convertBasicPackets(receivedPackets); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java index 6be5a27b49b..6c73167b162 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FloatField.java @@ -9,9 +9,10 @@ import com.yahoo.data.access.Inspector; /** - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias M\u00f8lster Lidal</a> + * @author Mathias Mølster Lidal */ public class FloatField extends DocsumField { + static final double EMPTY_VALUE = Float.NaN; public FloatField(String name) { @@ -26,10 +27,12 @@ public class FloatField extends DocsumField { } } + @Override public Object decode(ByteBuffer b) { return convert(b.getFloat()); } + @Override public Object decode(ByteBuffer b, FastHit hit) { Object field = decode(b); hit.setField(name, field); @@ -46,4 +49,5 @@ public class FloatField extends DocsumField { public Object convert(Inspector value) { return convert((float)value.asDouble(EMPTY_VALUE)); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java index fe44597b7d7..eef6fc73294 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/IntegerField.java @@ -12,9 +12,10 @@ import com.yahoo.search.result.NanNumber; import com.yahoo.data.access.Inspector; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a> + * @author Bjørn Borud */ public class IntegerField extends DocsumField { + static final int EMPTY_VALUE = Integer.MIN_VALUE; public IntegerField(String name) { @@ -25,20 +26,23 @@ public class IntegerField extends DocsumField { if (value == EMPTY_VALUE) { return NanNumber.NaN; } else { - return Integer.valueOf(value); + return value; } } + @Override public Object decode(ByteBuffer b) { return convert(b.getInt()); } + @Override public Object decode(ByteBuffer b, FastHit hit) { Object field = decode(b); hit.setField(name, field); return field; } + @Override public String toString() { return "field " + getName() + " type int"; } @@ -53,4 +57,5 @@ public class IntegerField extends DocsumField { public Object convert(Inspector value) { return convert((int)value.asLong(EMPTY_VALUE)); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java index 08cd4ddff35..9d22168485c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java @@ -86,4 +86,5 @@ public class LongdataField extends DocsumField implements VariableLengthField { public Object convert(Inspector value) { return convert(value.asData(Value.empty().asData())); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index eac1579a821..e5c9f048e53 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -55,8 +55,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { private static final CompoundName grouping=new CompoundName("grouping"); private static final CompoundName combinerows=new CompoundName("combinerows"); - /** If this is turned on this will fill summaries by dispatching directly to search nodes over RPC */ - private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); protected static final CompoundName PACKET_COMPRESSION_LIMIT = new CompoundName("packetcompressionlimit"); protected static final CompoundName PACKET_COMPRESSION_TYPE = new CompoundName("packetcompressiontype"); @@ -110,10 +108,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { protected abstract void doPartialFill(Result result, String summaryClass); - protected static boolean wantsRPCSummaryFill(Query query) { - return query.properties().getBoolean(dispatchSummaries); - } - /** * Returns whether we need to send the query when fetching summaries. * This is necessary if the query requests summary features or dynamic snippeting @@ -216,12 +210,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return new Result(query, ErrorMessage.createNullQuery(query.getHttpRequest().getUri().toString())); } - if (wantsRPCSummaryFill(query) && summaryNeedsQuery(query)) { - return new Result(query, ErrorMessage.createInvalidQueryParameter( - "When using dispatch.summaries and your summary/rankprofile require the query, " + - " you need to enable ranking.queryCache.")); - } - QueryRewrite.optimizeByRestrict(query); QueryRewrite.optimizeAndNot(query); QueryRewrite.collapseSingleComposites(query); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java index df8f6e92d57..6ca5feb610f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldPart.java @@ -4,11 +4,13 @@ package com.yahoo.prelude.hitfield; /** * Represents an element of a hit property * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public interface FieldPart { - public abstract boolean isFinal(); - public abstract boolean isToken(); - public abstract String getContent(); - public abstract String toString(); + + boolean isFinal(); + boolean isToken(); + String getContent(); + String toString(); + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java index 770ac24dcab..79d871d8c74 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java @@ -10,11 +10,10 @@ import com.yahoo.prelude.searcher.JuniperSearcher; import com.yahoo.text.XML; /** - * Represents a Field in a Hit. The original raw content and the field - * name cannot be modified. But the tokenized version can be retrieved - * and set. + * Represents a tokenized string field in a Hit. The original raw content and the field + * name cannot be modified. But the tokenized version can be retrieved and set. * - * @author <a href="mailto:larschr@yahoo-inc.com">Lars Christian Jensen</a> + * @author Lars Christian Jensen */ public class HitField { @@ -345,10 +344,7 @@ public class HitField { return xml.toString(); } - /** - * @return the content of this field, using the arguments as bolding - * tags, as an XML string - */ + /** Returns the content of this field, using the arguments as bolding tags, as an XML string */ public String quotedContent(String boldOpenTag, String boldCloseTag, String separatorTag, diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java index fa0ca62405f..268e1b53459 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/ImmutableFieldPart.java @@ -2,12 +2,12 @@ package com.yahoo.prelude.hitfield; /** - * Represents an element of a hit property which is a possibly - * mutable string element + * Represents an element of a hit property which is an immutable string element * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class ImmutableFieldPart implements FieldPart { + private final String content; private final String initContent; // Whether this element represents a (part of) a token or a @@ -15,10 +15,12 @@ public class ImmutableFieldPart implements FieldPart { // parts should inherit this state from the object they were // split from. private boolean tokenOrDelimiter; + public ImmutableFieldPart(String initContent, boolean tokenOrDelimiter) { this(initContent, initContent, tokenOrDelimiter); } + public ImmutableFieldPart(String initContent, String content, boolean tokenOrDelimiter) { @@ -27,9 +29,17 @@ public class ImmutableFieldPart implements FieldPart { this.content = content; this.tokenOrDelimiter = tokenOrDelimiter; } + + @Override public boolean isFinal() { return true; } + @Override public boolean isToken() { return tokenOrDelimiter; } + @Override public String getContent() { return content; } + public String getInitContent() { return initContent; } + + @Override public String toString() { return content; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java index bdd7cf04c3e..06db012309e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java @@ -19,7 +19,7 @@ import java.util.Iterator; /** * A JSON wrapper. Contains XML-style rendering of a JSON structure. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class JSONString implements Inspectable { diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java index 23ca8272851..17def6771ae 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/MarkupFieldPart.java @@ -4,19 +4,31 @@ package com.yahoo.prelude.hitfield; /** * Represents an element of a hit property which is markup, not content. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class MarkupFieldPart implements FieldPart { + private String content; + public MarkupFieldPart(String content) { this.content = content; } + + @Override public boolean isFinal() { return true; } + // Markup is never part of tokens as such + @Override public boolean isToken() { return false; } + public void setContent(String content) { this.content = content; } + + @Override public String getContent() { return content; } + + @Override public String toString() { return content; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java index dbbbca63d43..a0c9b10c519 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java @@ -6,8 +6,8 @@ package com.yahoo.prelude.hitfield; * * @author arnej27959 */ -public final class RawData -{ +public final class RawData { + private byte[] content; /** @@ -52,4 +52,5 @@ public final class RawData } return buf.toString(); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java index d7607fb7100..58018305fef 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/StringFieldPart.java @@ -5,9 +5,10 @@ package com.yahoo.prelude.hitfield; * Represents an element of a hit property which is a possibly * mutable string element * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class StringFieldPart implements FieldPart { + private String content; private final String initContent; // Whether this element represents a (part of) a token or a @@ -15,17 +16,28 @@ public class StringFieldPart implements FieldPart { // parts should inherit this state from the object they were // split from. private boolean tokenOrDelimiter; + public StringFieldPart(String content, boolean tokenOrDelimiter) { this.content = content; initContent = content; this.tokenOrDelimiter = tokenOrDelimiter; } + + @Override public boolean isFinal() { return false; } + + @Override public boolean isToken() { return tokenOrDelimiter; } + + @Override public String getContent() { return content; } + public void setContent(String content) { this.content = content; } public String getInitContent() { return initContent; } + + @Override public String toString() { return content; } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java index 8bafbfd6ab5..99c5daa05b8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java @@ -1,21 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.hitfield; -import com.yahoo.text.Utf8; import com.yahoo.text.XML; import com.yahoo.data.access.Inspector; -import com.yahoo.data.access.Inspectable; import com.yahoo.data.access.Type; -import com.yahoo.data.access.simple.Value; -import com.yahoo.data.access.slime.SlimeAdapter; -import java.nio.charset.StandardCharsets; - -import java.util.Iterator; import java.util.Map; /** * Utility class for converting accessible data into the historical "prelude" xml format. - **/ + */ public class XmlRenderer { public static StringBuilder render(StringBuilder target, Inspector value) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 1b775d2c46f..1b6ba5cc085 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -9,6 +9,7 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.protect.Error; +import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.slime.ArrayTraverser; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.prelude.fastsearch.FS4ResourcePool; @@ -22,12 +23,9 @@ import com.yahoo.search.result.Hit; import com.yahoo.data.access.Inspector; import com.yahoo.slime.BinaryFormat; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.search.DispatchConfig; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -38,12 +36,15 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * A dispatcher communicates with search nodes to (in the future) perform queries and (now) fill hits. + * A dispatcher communicates with search nodes to perform queries and fill hits. + * + * This is currently not functionally complete: Queries can only be dispatched to a single node, + * and summaries can only be requested when they do not need the query. + * * This class is multithread safe. * * @author bratseth */ -@Beta public class Dispatcher extends AbstractComponent { private final static Logger log = Logger.getLogger(Dispatcher.class.getName()); @@ -82,15 +83,21 @@ public class Dispatcher extends AbstractComponent { public SearchCluster searchCluster() { return searchCluster; } /** Fills the given summary class by sending RPC requests to the right search nodes */ - public void fill(Result result, String summaryClass, CompressionType compression) { + public void fill(Result result, String summaryClass, DocumentDatabase documentDb, CompressionType compression) { try { ListMap<Integer, FastHit> hitsByNode = hitsByNode(result); + if (result.getQuery().getTraceLevel() >=3) + result.getQuery().trace("Sending " + hitsByNode.size() + " summary fetch RPC requests", 3); + GetDocsumsResponseReceiver responseReceiver = new GetDocsumsResponseReceiver(hitsByNode.size(), compressor, result); for (Map.Entry<Integer, List<FastHit>> nodeHits : hitsByNode.entrySet()) { sendGetDocsumsRequest(nodeHits.getKey(), nodeHits.getValue(), summaryClass, compression, result, responseReceiver); } - responseReceiver.processResponses(result.getQuery(), summaryClass); + responseReceiver.processResponses(result.getQuery(), summaryClass, documentDb); + result.hits().setFilled(summaryClass); + result.hits().setSorted(false); + result.analyzeHits(); } catch (TimeoutException e) { result.hits().addError(ErrorMessage.createTimeout("Summary data is incomplete: " + e.getMessage())); @@ -192,7 +199,7 @@ public class Dispatcher extends AbstractComponent { * Call this from the dispatcher thread to initiate and complete processing of responses. * This will block until all responses are available and processed, or to timeout. */ - public void processResponses(Query query, String summaryClass) throws TimeoutException { + public void processResponses(Query query, String summaryClass, DocumentDatabase documentDb) throws TimeoutException { try { int skippedHits = 0; while (outstandingResponses > 0) { @@ -203,11 +210,12 @@ public class Dispatcher extends AbstractComponent { Client.GetDocsumsResponseOrError response = responses.poll(timeLeftMs, TimeUnit.MILLISECONDS); if (response == null) throwTimeout(); - skippedHits += processResponse(response); + skippedHits += processResponse(response, summaryClass, documentDb); outstandingResponses--; } if (skippedHits != 0) { - result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit summary data for summary " + summaryClass + " for " + skippedHits + " hits")); + result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit summary data for summary " + + summaryClass + " for " + skippedHits + " hits")); } } catch (InterruptedException e) { @@ -215,7 +223,9 @@ public class Dispatcher extends AbstractComponent { } } - private int processResponse(Client.GetDocsumsResponseOrError responseOrError) { + private int processResponse(Client.GetDocsumsResponseOrError responseOrError, + String summaryClass, + DocumentDatabase documentDb) { if (responseOrError.error().isPresent()) { if (hasReportedError) return 0; String error = responseOrError.error().get(); @@ -226,7 +236,7 @@ public class Dispatcher extends AbstractComponent { Client.GetDocsumsResponse response = responseOrError.response().get(); CompressionType compression = CompressionType.valueOf(response.compression()); byte[] slimeBytes = compressor.decompress(response.compressedSlimeBytes(), compression, response.uncompressedSize()); - return fill(response.hitsContext(), slimeBytes); + return fill(response.hitsContext(), summaryClass, documentDb, slimeBytes); } return 0; } @@ -241,7 +251,7 @@ public class Dispatcher extends AbstractComponent { }); } - private int fill(List<FastHit> hits, byte[] slimeBytes) { + private int fill(List<FastHit> hits, String summaryClass, DocumentDatabase documentDb, byte[] slimeBytes) { com.yahoo.slime.Inspector root = BinaryFormat.decode(slimeBytes).get(); com.yahoo.slime.Inspector errors = root.field("errors"); boolean hasErrors = errors.valid() && (errors.entries() > 0); @@ -256,7 +266,7 @@ public class Dispatcher extends AbstractComponent { for (int i = 0; i < hits.size(); i++) { Inspector summary = summaries.entry(i).field("docsum"); if (summary.fieldCount() != 0) { - fill(hits.get(i), summary); + hits.get(i).addSummary(documentDb.getDocsumDefinitionSet().getDocsumDefinition(summaryClass), summary); } else { skippedHits++; } @@ -264,27 +274,6 @@ public class Dispatcher extends AbstractComponent { return skippedHits; } - private void fill(FastHit hit, Inspector summary) { - hit.reserve(summary.fieldCount()); - summary.traverse((String name, Inspector value) -> { - hit.setField(name, nativeTypeOf(value)); - }); - } - - private Object nativeTypeOf(Inspector inspector) { - switch (inspector.type()) { - case ARRAY: return inspector; - case OBJECT: return inspector; - case BOOL: return inspector.asBool(); - case DATA: return inspector.asData(); - case DOUBLE: return inspector.asDouble(); - case LONG: return inspector.asLong(); - case STRING: return inspector.asString(); // TODO: Keep as utf8 - case EMPTY : return null; - default: throw new IllegalArgumentException("Unexpected Slime type " + inspector.type()); - } - } - } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java index 60f2691aa69..9b2a24cd01f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java @@ -41,7 +41,6 @@ import java.util.stream.Collectors; * * @author bratseth */ -@Beta public class SearchCluster implements NodeManager<SearchCluster.Node> { private static final Logger log = Logger.getLogger(SearchCluster.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java index aa2de1a1298..2f5d833a5be 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java @@ -4,16 +4,17 @@ package com.yahoo.search.grouping.request; /** * This class represents a constant {@link Long} value in a {@link GroupingExpression}. * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class LongValue extends ConstantValue<Long> { /** * Constructs a new instance of this class. * - * @param value The immutable value to assign to this. + * @param value the immutable value to assign to this. */ public LongValue(long value) { super(value); } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index 6adbac56dbe..026538eab54 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -403,16 +403,13 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi //TODO Should it be deprecated ? public final Map<String,Object> fields() { return getUnmodifiableFieldMap(); } - /** - * Will preallocate in order to avoid resizing. - * @param minSize The minimum size to reserve - */ + /** Aallocate room for the given number of fields to avoid resizing. */ public void reserve(int minSize) { getFieldMap(minSize); } /** - * Fields + * Returns an iterator over the fields of this * * @return an iterator for traversing the fields of this hit */ @@ -663,9 +660,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi if (p == null) { return null; } else if (p instanceof HitField) { - HitField hf = (HitField) p; - - return hf.quotedContent(false); + return ((HitField)p).quotedContent(false); } else if (p instanceof StructuredData) { return p.toString(); } else if (p instanceof XMLString || p instanceof JSONString) { @@ -743,9 +738,10 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } /** - * For vespa internal use only. + * For internal use only. * Gives access to the modifiable backing set of filled summaries. * This set might be unmodifiable if the size is less than or equal to 1 + * * @return the set of filled summaries. */ protected final Set<String> getFilledInternal() { @@ -755,6 +751,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi private Map<String,Object> getFieldMap() { return getFieldMap(16); } + private Map<String,Object> getFieldMap(int minSize) { if (fields == null) { // Compensate for loadfactor and then some, rounded up.... diff --git a/container-search/src/main/java/com/yahoo/search/result/NanNumber.java b/container-search/src/main/java/com/yahoo/search/result/NanNumber.java index 2103583dfa0..078ac04f85e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/NanNumber.java +++ b/container-search/src/main/java/com/yahoo/search/result/NanNumber.java @@ -4,14 +4,14 @@ package com.yahoo.search.result; /** * A class representing unset or undefined numeric values. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ @SuppressWarnings("serial") public final class NanNumber extends Number { + public static final NanNumber NaN = new NanNumber(); - private NanNumber() { - } + private NanNumber() { } @Override public double doubleValue() { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 20150fc2671..3cfd8b337fc 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.fastsearch.test; import com.google.common.collect.ImmutableList; import com.yahoo.component.chain.Chain; import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.container.handler.VipStatus; import com.yahoo.container.search.Fs4Config; import com.yahoo.fs4.mplex.*; import com.yahoo.fs4.test.QueryTestCase; @@ -14,6 +15,7 @@ import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.container.protect.Error; import com.yahoo.fs4.*; import com.yahoo.prelude.fastsearch.test.fs4mock.MockBackend; +import com.yahoo.prelude.fastsearch.test.fs4mock.MockFS4ResourcePool; import com.yahoo.prelude.fastsearch.test.fs4mock.MockFSChannel; import com.yahoo.processing.execution.Execution.Trace; import com.yahoo.search.Query; @@ -111,30 +113,45 @@ public class FastSearcherTestCase { .summaryclass(new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder().name("simple").id(7)) .rankprofile(new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder() .name("simpler").hasRankFeatures(false).hasSummaryFeatures(false)))); + + List<SearchCluster.Node> nodes = new ArrayList<>(); + nodes.add(new SearchCluster.Node("host1", 5000, 0)); + nodes.add(new SearchCluster.Node("host2", 5000, 0)); + + MockFS4ResourcePool mockFs4ResourcePool = new MockFS4ResourcePool(); FastSearcher fastSearcher = new FastSearcher(new MockBackend(), - new FS4ResourcePool(1), - new MockDispatcher(Collections.emptyList()), + mockFs4ResourcePool, + new MockDispatcher(nodes, mockFs4ResourcePool, 1, new VipStatus()), new SummaryParameters(null), new ClusterParams("testhittype"), - new CacheParams(100, 1e64), + new CacheParams(0, 0), documentdbConfigWithOneDb); - String query = "?query=sddocname:a&dispatch.summaries"; - Result result = doSearch(fastSearcher,new Query(query), 0, 10); - ErrorMessage message = result.hits().getError(); - - assertNotNull("Got error", message); - assertEquals("Invalid query parameter", message.getMessage()); - assertEquals("When using dispatch.summaries and your summary/rankprofile require the query, you need to enable ranking.queryCache.", message.getDetailedMessage()); - assertEquals(Error.INVALID_QUERY_PARAMETER.code, message.getCode()); + { // No direct.summaries + String query = "?query=sddocname:a&summary=simple"; + Result result = doSearch(fastSearcher, new Query(query), 0, 10); + doFill(fastSearcher, result); + ErrorMessage error = result.hits().getError(); + assertNull("Since we don't route to the dispatcher we hit the mock backend, so no error", error); + } - query = "?query=sddocname:a&dispatch.summaries&ranking.queryCache"; - result = doSearch(fastSearcher,new Query(query), 0, 10); - assertNull(result.hits().getError()); + { // direct.summaries due to query cache + String query = "?query=sddocname:a&ranking.queryCache"; + Result result = doSearch(fastSearcher, new Query(query), 0, 10); + doFill(fastSearcher, result); + ErrorMessage error = result.hits().getError(); + assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used", + "Error response from rpc node connection to host1:0: Connection error", error.getDetailedMessage()); + } - query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler"; - result = doSearch(fastSearcher,new Query(query), 0, 10); - assertNull(result.hits().getError()); + { // direct.summaries due to no summary features + String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler"; + Result result = doSearch(fastSearcher, new Query(query), 0, 10); + doFill(fastSearcher, result); + ErrorMessage error = result.hits().getError(); + assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used", + "Error response from rpc node connection to host1:0: Connection error", error.getDetailedMessage()); + } } @Test @@ -281,7 +298,7 @@ public class FastSearcherTestCase { assertEquals(100, fastSearcher.getCacheControl().capacity()); // Default cache =100MB - Query query = new Query("?query=ignored"); + Query query = new Query("?query=ignored&dispatch.summaries=false"); query.getRanking().setQueryCache(true); Result result = doSearch(fastSearcher, query, 0, 10); @@ -321,7 +338,6 @@ public class FastSearcherTestCase { answer.flip(); answer.get(expected); - assertEquals(expected.length, actual.length); for (int i = 0; i < expected.length; ++i) { if (expected[i] == IGNORE) { actual[i] = IGNORE; diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java index 8d705406774..37a74831e56 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java @@ -96,13 +96,13 @@ public class MockFSChannel extends FS4Channel { && lastQueryPacket.getLastOffset() >= 1) { result.addDocument( new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(123), - 2003, 234, 1000)); + 2003, 234, 0)); } if (lastQueryPacket.getOffset() <= 1 && lastQueryPacket.getLastOffset() >= 2) { result.addDocument( new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(456), - 1855, 234, 1001)); + 1855, 234, 1)); } packets.add(result); } diff --git a/container-search/src/test/java/com/yahoo/prelude/test/NullSetMemberTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/NullSetMemberTestCase.java index f66acb2bdc0..f900b4c9d8a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/NullSetMemberTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/NullSetMemberTestCase.java @@ -5,10 +5,14 @@ import org.junit.Test; import java.util.HashSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * Tests null members in HashSet */ -public class NullSetMemberTestCase extends junit.framework.TestCase { +public class NullSetMemberTestCase { @Test public void testNullMember() { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java b/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java index 9938498fa1a..ed421be1947 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java @@ -2,6 +2,11 @@ package com.yahoo.search.dispatch; import com.yahoo.compress.CompressionType; +import com.yahoo.log.event.Collection; +import com.yahoo.prelude.fastsearch.DocsumDefinition; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; +import com.yahoo.prelude.fastsearch.DocsumField; +import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -10,7 +15,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; @@ -44,7 +52,8 @@ public class FillTestCase { client.setDocsumReponse("host1", 2, "summaryClass1", map("field1", "s.1.2", "field2", 2)); client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); client.setDocsumReponse("host0", 4, "summaryClass1", map("field1", "s.0.4", "field2", 4)); - dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); + + dispatcher.fill(result, "summaryClass1", db(), CompressionType.valueOf("LZ4")); assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); @@ -79,7 +88,7 @@ public class FillTestCase { client.setDocsumReponse("host1", 2, "summaryClass1", new HashMap<>()); client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); client.setDocsumReponse("host0", 4, "summaryClass1",new HashMap<>()); - dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); + dispatcher.fill(result, "summaryClass1", db(), CompressionType.valueOf("LZ4")); assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); @@ -108,11 +117,20 @@ public class FillTestCase { Result result = new Result(query); result.hits().add(createHit(0, 0)); - dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); + dispatcher.fill(result, "summaryClass1", db(), CompressionType.valueOf("LZ4")); assertEquals("Malfunctioning", result.hits().getError().getDetailedMessage()); } + private DocumentDatabase db() { + List<DocsumField> fields = new ArrayList<>(); + fields.add(DocsumField.create("field1", "string")); + fields.add(DocsumField.create("field2", "int64")); + DocsumDefinitionSet docsums = new DocsumDefinitionSet(Collections.singleton(new DocsumDefinition("summaryClass1", + fields))); + return new DocumentDatabase("default", docsums, Collections.emptySet()); + } + private FastHit createHit(int sourceNodeId, int hitId) { FastHit hit = new FastHit("hit:" + hitId, 1.0); hit.setPartId(sourceNodeId, 0); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index 4604678198d..f3133855729 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -9,24 +9,32 @@ import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import org.junit.Test; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Tests untyped query profiles * * @author bratseth */ -public class QueryProfileTestCase extends junit.framework.TestCase { +public class QueryProfileTestCase { + @Test public void testBasics() { - QueryProfile profile=new QueryProfile("test"); - profile.set("a","a-value", (QueryProfileRegistry)null); - profile.set("b.c","b.c-value", (QueryProfileRegistry)null); - profile.set("d.e.f","d.e.f-value", (QueryProfileRegistry)null); + QueryProfile profile = new QueryProfile("test"); + profile.set("a","a-value", null); + profile.set("b.c","b.c-value", null); + profile.set("d.e.f","d.e.f-value", null); CompiledQueryProfile cprofile = profile.compile(null); @@ -42,10 +50,11 @@ public class QueryProfileTestCase extends junit.framework.TestCase { } /** Tests cloning, with wrappers used in production in place */ + @Test public void testCloning() { QueryProfile classProfile=new QueryProfile("test"); - classProfile.set("a","aValue", (QueryProfileRegistry)null); - classProfile.set("b",3, (QueryProfileRegistry)null); + classProfile.set("a","aValue", null); + classProfile.set("b",3, null); Properties properties = new QueryProfileProperties(classProfile.compile(null)); @@ -57,11 +66,12 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("aValue",propertiesClone.get("a")); } + @Test public void testFreezing() { QueryProfile profile=new QueryProfile("test"); - profile.set("a","a-value", (QueryProfileRegistry)null); - profile.set("b.c","b.c-value", (QueryProfileRegistry)null); - profile.set("d.e.f","d.e.f-value", (QueryProfileRegistry)null); + profile.set("a","a-value", null); + profile.set("b.c","b.c-value", null); + profile.set("d.e.f","d.e.f-value", null); assertFalse(profile.isFrozen()); assertEquals("a-value",profile.get("a")); @@ -73,7 +83,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertTrue(((QueryProfile)profile.lookup("d.e",null)).isFrozen()); try { - profile.set("a","value", (QueryProfileRegistry)null); + profile.set("a","value", null); fail("Expected exception"); } catch (IllegalStateException e) { @@ -89,6 +99,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { } + @Test public void testGetSubObjects() { QueryProfile barn=new QueryProfile("barn"); QueryProfile mor=new QueryProfile("mor"); @@ -101,11 +112,11 @@ public class QueryProfileTestCase extends junit.framework.TestCase { far.addInherited(farfar); barn.addInherited(mor); barn.addInherited(far); - mormor.set("a.mormor","a.mormor", (QueryProfileRegistry)null); - barn.set("a.barn","a.barn", (QueryProfileRegistry)null); - mor.set("b.mor", "b.mor", (QueryProfileRegistry)null); - far.set("b.far", "b.far", (QueryProfileRegistry)null); - far.set("a.far","a.far", (QueryProfileRegistry)null); + mormor.set("a.mormor","a.mormor", null); + barn.set("a.barn","a.barn", null); + mor.set("b.mor", "b.mor", null); + far.set("b.far", "b.far", null); + far.set("a.far","a.far", null); CompiledQueryProfile cbarn = barn.compile(null); assertSameObjects(cbarn, "a", Arrays.asList("mormor","far","barn")); @@ -114,6 +125,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("b.far", cbarn.get("b.far")); } + @Test public void testInheritance() { QueryProfile barn=new QueryProfile("barn"); QueryProfile mor=new QueryProfile("mor"); @@ -127,24 +139,24 @@ public class QueryProfileTestCase extends junit.framework.TestCase { mor.addInherited(morfar); far.addInherited(farfar); - morfar.set("a","morfar-a", (QueryProfileRegistry)null); - mormor.set("a","mormor-a", (QueryProfileRegistry)null); - farfar.set("a","farfar-a", (QueryProfileRegistry)null); - mor.set("a","mor-a", (QueryProfileRegistry)null); - far.set("a","far-a", (QueryProfileRegistry)null); - barn.set("a","barn-a", (QueryProfileRegistry)null); + morfar.set("a","morfar-a", null); + mormor.set("a","mormor-a", null); + farfar.set("a","farfar-a", null); + mor.set("a","mor-a", null); + far.set("a","far-a", null); + barn.set("a","barn-a", null); - mormor.set("b","mormor-b", (QueryProfileRegistry)null); - far.set("b","far-b", (QueryProfileRegistry)null); + mormor.set("b","mormor-b", null); + far.set("b","far-b", null); - mor.set("c","mor-c", (QueryProfileRegistry)null); - far.set("c","far-c", (QueryProfileRegistry)null); + mor.set("c","mor-c", null); + far.set("c","far-c", null); - mor.set("d.a","mor-d.a", (QueryProfileRegistry)null); - barn.set("d.b","barn-d.b", (QueryProfileRegistry)null); + mor.set("d.a","mor-d.a", null); + barn.set("d.b","barn-d.b", null); QueryProfile annetBarn=new QueryProfile("annetBarn"); - annetBarn.set("venn",barn, (QueryProfileRegistry)null); + annetBarn.set("venn",barn, null); CompiledQueryProfile cbarn = barn.compile(null); CompiledQueryProfile cannetBarn = annetBarn.compile(null); @@ -161,6 +173,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("mor-d.a", cbarn.get("d.a")); } + @Test public void testInheritance2Level() { QueryProfile barn=new QueryProfile("barn"); QueryProfile mor=new QueryProfile("mor"); @@ -174,24 +187,24 @@ public class QueryProfileTestCase extends junit.framework.TestCase { mor.addInherited(morfar); far.addInherited(farfar); - morfar.set("a.x","morfar-a", (QueryProfileRegistry)null); - mormor.set("a.x","mormor-a", (QueryProfileRegistry)null); - farfar.set("a.x","farfar-a", (QueryProfileRegistry)null); - mor.set("a.x","mor-a", (QueryProfileRegistry)null); - far.set("a.x","far-a", (QueryProfileRegistry)null); - barn.set("a.x","barn-a", (QueryProfileRegistry)null); + morfar.set("a.x","morfar-a", null); + mormor.set("a.x","mormor-a", null); + farfar.set("a.x","farfar-a", null); + mor.set("a.x","mor-a", null); + far.set("a.x","far-a", null); + barn.set("a.x","barn-a", null); - mormor.set("b.x","mormor-b", (QueryProfileRegistry)null); - far.set("b.x","far-b", (QueryProfileRegistry)null); + mormor.set("b.x","mormor-b", null); + far.set("b.x","far-b", null); - mor.set("c.x","mor-c", (QueryProfileRegistry)null); - far.set("c.x","far-c", (QueryProfileRegistry)null); + mor.set("c.x","mor-c", null); + far.set("c.x","far-c", null); - mor.set("d.a.x","mor-d.a", (QueryProfileRegistry)null); - barn.set("d.b.x","barn-d.b", (QueryProfileRegistry)null); + mor.set("d.a.x","mor-d.a", null); + barn.set("d.b.x","barn-d.b", null); QueryProfile annetBarn=new QueryProfile("annetBarn"); - annetBarn.set("venn",barn, (QueryProfileRegistry)null); + annetBarn.set("venn",barn, null); CompiledQueryProfile cbarn = barn.compile(null); CompiledQueryProfile cannetBarn = annetBarn.compile(null); @@ -208,6 +221,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("mor-d.a", cbarn.get("d.a.x")); } + @Test public void testInheritance3Level() { QueryProfile barn=new QueryProfile("barn"); QueryProfile mor=new QueryProfile("mor"); @@ -221,24 +235,24 @@ public class QueryProfileTestCase extends junit.framework.TestCase { mor.addInherited(morfar); far.addInherited(farfar); - morfar.set("y.a.x","morfar-a", (QueryProfileRegistry)null); - mormor.set("y.a.x","mormor-a", (QueryProfileRegistry)null); - farfar.set("y.a.x","farfar-a", (QueryProfileRegistry)null); - mor.set("y.a.x","mor-a", (QueryProfileRegistry)null); - far.set("y.a.x","far-a", (QueryProfileRegistry)null); - barn.set("y.a.x","barn-a", (QueryProfileRegistry)null); + morfar.set("y.a.x","morfar-a", null); + mormor.set("y.a.x","mormor-a", null); + farfar.set("y.a.x","farfar-a", null); + mor.set("y.a.x","mor-a", null); + far.set("y.a.x","far-a", null); + barn.set("y.a.x","barn-a", null); - mormor.set("y.b.x","mormor-b", (QueryProfileRegistry)null); - far.set("y.b.x","far-b", (QueryProfileRegistry)null); + mormor.set("y.b.x","mormor-b", null); + far.set("y.b.x","far-b", null); - mor.set("y.c.x","mor-c", (QueryProfileRegistry)null); - far.set("y.c.x","far-c", (QueryProfileRegistry)null); + mor.set("y.c.x","mor-c", null); + far.set("y.c.x","far-c", null); - mor.set("y.d.a.x","mor-d.a", (QueryProfileRegistry)null); - barn.set("y.d.b.x","barn-d.b", (QueryProfileRegistry)null); + mor.set("y.d.a.x","mor-d.a", null); + barn.set("y.d.b.x","barn-d.b", null); QueryProfile annetBarn=new QueryProfile("annetBarn"); - annetBarn.set("venn",barn, (QueryProfileRegistry)null); + annetBarn.set("venn",barn, null); CompiledQueryProfile cbarn = barn.compile(null); CompiledQueryProfile cannetBarn = annetBarn.compile(null); @@ -255,6 +269,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("mor-d.a", cbarn.get("y.d.a.x")); } + @Test public void testListProperties() { QueryProfile barn=new QueryProfile("barn"); QueryProfile mor=new QueryProfile("mor"); @@ -268,18 +283,18 @@ public class QueryProfileTestCase extends junit.framework.TestCase { mor.addInherited(morfar); far.addInherited(farfar); - morfar.set("a","morfar-a", (QueryProfileRegistry)null); - morfar.set("model.b","morfar-model.b", (QueryProfileRegistry)null); - mormor.set("a","mormor-a", (QueryProfileRegistry)null); - mormor.set("model.b","mormor-model.b", (QueryProfileRegistry)null); - farfar.set("a","farfar-a", (QueryProfileRegistry)null); - mor.set("a","mor-a", (QueryProfileRegistry)null); - far.set("a","far-a", (QueryProfileRegistry)null); - barn.set("a","barn-a", (QueryProfileRegistry)null); - mormor.set("b","mormor-b", (QueryProfileRegistry)null); - far.set("b","far-b", (QueryProfileRegistry)null); - mor.set("c","mor-c", (QueryProfileRegistry)null); - far.set("c","far-c", (QueryProfileRegistry)null); + morfar.set("a","morfar-a", null); + morfar.set("model.b","morfar-model.b", null); + mormor.set("a","mormor-a", null); + mormor.set("model.b","mormor-model.b", null); + farfar.set("a","farfar-a", null); + mor.set("a","mor-a", null); + far.set("a","far-a", null); + barn.set("a","barn-a", null); + mormor.set("b","mormor-b", null); + far.set("b","far-b", null); + mor.set("c","mor-c", null); + far.set("c","far-c", null); CompiledQueryProfile cbarn = barn.compile(null); @@ -308,9 +323,10 @@ public class QueryProfileTestCase extends junit.framework.TestCase { } /** Tests that dots are followed when setting overridability */ + @Test public void testInstanceOverridable() { QueryProfile profile=new QueryProfile("root/unoverridableIndex"); - profile.set("model.defaultIndex","default", (QueryProfileRegistry)null); + profile.set("model.defaultIndex","default", null); profile.setOverridable("model.defaultIndex",false,null); assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null).booleanValue()); @@ -326,10 +342,11 @@ public class QueryProfileTestCase extends junit.framework.TestCase { } /** Tests that dots are followed when setting overridability...also with variants */ + @Test public void testInstanceOverridableWithVariants() { QueryProfile profile=new QueryProfile("root/unoverridableIndex"); profile.setDimensions(new String[] {"x"}); - profile.set("model.defaultIndex","default", (QueryProfileRegistry)null); + profile.set("model.defaultIndex","default", null); profile.setOverridable("model.defaultIndex",false,null); assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null)); @@ -344,10 +361,11 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("de",query.getModel().getLanguage().languageCode()); } + @Test public void testSimpleInstanceOverridableWithVariants1() { QueryProfile profile=new QueryProfile("test"); profile.setDimensions(new String[] {"x"}); - profile.set("a","original", (QueryProfileRegistry)null); + profile.set("a","original", null); profile.setOverridable("a",false,null); assertFalse(profile.isDeclaredOverridable("a",null)); @@ -356,6 +374,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("original",query.properties().get("a")); } + @Test public void testSimpleInstanceOverridableWithVariants2() { QueryProfile profile=new QueryProfile("test"); profile.setDimensions(new String[] {"x"}); @@ -369,39 +388,43 @@ public class QueryProfileTestCase extends junit.framework.TestCase { } /** Tests having both an explicit reference and an override */ + @Test public void testExplicitReferenceOverride() { QueryProfile a1=new QueryProfile("a1"); - a1.set("b","a1.b", (QueryProfileRegistry)null); + a1.set("b","a1.b", null); QueryProfile profile=new QueryProfile("test"); - profile.set("a",a1, (QueryProfileRegistry)null); - profile.set("a.b","a.b", (QueryProfileRegistry)null); + profile.set("a",a1, null); + profile.set("a.b","a.b", null); assertEquals("a.b",profile.compile(null).get("a.b")); } + @Test public void testSettingNonLeaf1() { QueryProfile p=new QueryProfile("test"); - p.set("a","a-value", (QueryProfileRegistry)null); - p.set("a.b","a.b-value", (QueryProfileRegistry)null); + p.set("a","a-value", null); + p.set("a.b","a.b-value", null); QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); assertEquals("a-value", cp.get("a")); assertEquals("a.b-value", cp.get("a.b")); } + @Test public void testSettingNonLeaf2() { QueryProfile p=new QueryProfile("test"); - p.set("a.b","a.b-value", (QueryProfileRegistry)null); - p.set("a","a-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value", null); + p.set("a","a-value", null); QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); assertEquals("a-value", cp.get("a")); assertEquals("a.b-value", cp.get("a.b")); } + @Test public void testSettingNonLeaf3a() { QueryProfile p=new QueryProfile("test"); p.setDimensions(new String[] {"x"}); - p.set("a.b","a.b-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value", null); p.set("a","a-value",new String[] {"x1"}, null); QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); @@ -412,11 +435,12 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("a.b-value", cp.get("a.b", new String[] {"x1"})); } + @Test public void testSettingNonLeaf3b() { QueryProfile p=new QueryProfile("test"); p.setDimensions(new String[] {"x"}); p.set("a","a-value",new String[] {"x1"}, null); - p.set("a.b","a.b-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value", null); QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); @@ -426,11 +450,12 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("a.b-value", cp.get("a.b",new String[] {"x1"})); } + @Test public void testSettingNonLeaf4a() { QueryProfile p=new QueryProfile("test"); p.setDimensions(new String[] {"x"}); p.set("a.b","a.b-value",new String[] {"x1"}, null); - p.set("a","a-value", (QueryProfileRegistry)null); + p.set("a","a-value", null); QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); @@ -454,6 +479,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); } + @Test public void testSettingNonLeaf5() { QueryProfile p=new QueryProfile("test"); p.setDimensions(new String[] {"x"}); @@ -468,20 +494,22 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); } + @Test public void testListingWithNonLeafs() { QueryProfile p=new QueryProfile("test"); - p.set("a","a-value", (QueryProfileRegistry)null); - p.set("a.b","a.b-value", (QueryProfileRegistry)null); + p.set("a","a-value", null); + p.set("a.b","a.b-value", null); Map<String,Object> values = p.compile(null).listValues("a"); assertEquals(1,values.size()); assertEquals("a.b-value",values.get("b")); } + @Test public void testRankTypeNames() { QueryProfile p=new QueryProfile("test"); - p.set("a.$b","foo", (QueryProfileRegistry)null); - p.set("a.query(b)","bar", (QueryProfileRegistry)null); - p.set("a.b.default-index","fuu", (QueryProfileRegistry)null); + p.set("a.$b","foo", null); + p.set("a.query(b)","bar", null); + p.set("a.b.default-index","fuu", null); CompiledQueryProfile cp = p.compile(null); assertEquals("foo", cp.get("a.$b")); @@ -499,9 +527,10 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("fuu", p2.get("b.default-index")); } + @Test public void testQueryProfileInlineValueReassignment() { QueryProfile p=new QueryProfile("test"); - p.set("source.rel.params.query","%{model.queryString}", (QueryProfileRegistry)null); + p.set("source.rel.params.query","%{model.queryString}", null); p.freeze(); Query q = new Query(HttpRequest.createTestRequest("?query=foo", Method.GET), p.compile(null)); assertEquals("foo",q.properties().get("source.rel.params.query")); @@ -511,9 +540,10 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("foo",q.properties().listProperties().get("source.rel.params.query")); // Is still foo because model variables are not supported with the list function } + @Test public void testQueryProfileInlineValueReassignmentSimpleName() { QueryProfile p=new QueryProfile("test"); - p.set("key","%{model.queryString}", (QueryProfileRegistry)null); + p.set("key","%{model.queryString}", null); p.freeze(); Query q = new Query(HttpRequest.createTestRequest("?query=foo", Method.GET), p.compile(null)); assertEquals("foo",q.properties().get("key")); @@ -523,9 +553,10 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("foo",q.properties().listProperties().get("key")); // Is still bar because model variables are not supported with the list function } + @Test public void testQueryProfileInlineValueReassignmentSimpleNameGenericProperty() { QueryProfile p=new QueryProfile("test"); - p.set("key","%{value}", (QueryProfileRegistry)null); + p.set("key","%{value}", null); p.freeze(); Query q = new Query(HttpRequest.createTestRequest("?query=test&value=foo", Method.GET), p.compile(null)); assertEquals("foo",q.properties().get("key")); @@ -535,6 +566,7 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("bar",q.properties().listProperties().get("key")); } + @Test public void testQueryProfileModelValueListing() { QueryProfile p=new QueryProfile("test"); p.freeze(); @@ -546,11 +578,12 @@ public class QueryProfileTestCase extends junit.framework.TestCase { assertEquals("bar",q.properties().listProperties().get("model.queryString")); // Is still bar because model variables are not supported with the list function } + @Test public void testEmptyBoolean() { QueryProfile p=new QueryProfile("test"); p.setDimensions(new String[] {"x","y"}); - p.set("clustering.something","bar", (QueryProfileRegistry)null); - p.set("clustering.something","bar",new String[] {"x1","y1"}, null); + p.set("clustering.something","bar", null); + p.set("clustering.something","bar", new String[] {"x1","y1"}, null); p.freeze(); Query q = new Query(HttpRequest.createTestRequest("?x=x1&y=y1&query=bar&clustering.timeline.kano=tur&" + "clustering.enable=true&clustering.timeline.bucketspec=-" + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java index 9e920799bbc..9a7a57d5e10 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -10,20 +10,24 @@ import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import org.junit.Test; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.junit.Assert.assertEquals; + /** * @author bratseth */ -public class QueryProfileVariantsTestCase extends junit.framework.TestCase { +public class QueryProfileVariantsTestCase { + @Test public void testSimple() { QueryProfile profile=new QueryProfile("a"); - profile.set("a","a.deflt", (QueryProfileRegistry)null); + profile.set("a","a.deflt", null); profile.setDimensions(new String[] {"x","y","z"}); profile.set("a","a.1.*.*",new String[] {"x1",null,null}, null); profile.set("a","a.1.*.1",new String[] {"x1",null,"z1"}, null); @@ -63,10 +67,11 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertGet("a.2.*.*","a",new String[] {"x2","y?","z?"}, profile, cprofile); } + @Test public void testVariantsOfInlineCompound() { QueryProfile profile=new QueryProfile("test"); profile.setDimensions(new String[] {"x"}); - profile.set("a.b","a.b", (QueryProfileRegistry)null); + profile.set("a.b","a.b", null); profile.set("a.b","a.b.x1",new String[] {"x1"}, null); profile.set("a.b","a.b.x2",new String[] {"x2"}, null); @@ -77,13 +82,14 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("a.b.x2",cprofile.get("a.b", toMap("x=x2"))); } + @Test public void testVariantsOfExplicitCompound() { QueryProfile a1=new QueryProfile("a1"); - a1.set("b","a.b", (QueryProfileRegistry)null); + a1.set("b","a.b", null); QueryProfile profile=new QueryProfile("test"); profile.setDimensions(new String[] {"x"}); - profile.set("a",a1, (QueryProfileRegistry)null); + profile.set("a",a1, null); profile.set("a.b","a.b.x1",new String[] {"x1"}, null); profile.set("a.b","a.b.x2",new String[] {"x2"}, null); @@ -94,6 +100,7 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("a.b.x2",cprofile.get("a.b", toMap("x=x2"))); } + @Test public void testCompound() { // Configuration phase @@ -101,22 +108,22 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { profile.setDimensions(new String[] {"x","y"}); QueryProfile a1=new QueryProfile("a1"); - a1.set("b","a1.b.default", (QueryProfileRegistry)null); - a1.set("c","a1.c.default", (QueryProfileRegistry)null); - a1.set("d","a1.d.default", (QueryProfileRegistry)null); - a1.set("e","a1.e.default", (QueryProfileRegistry)null); + a1.set("b","a1.b.default", null); + a1.set("c","a1.c.default", null); + a1.set("d","a1.d.default", null); + a1.set("e","a1.e.default", null); QueryProfile a2=new QueryProfile("a2"); - a2.set("b","a2.b.default", (QueryProfileRegistry)null); - a2.set("c","a2.c.default", (QueryProfileRegistry)null); - a2.set("d","a2.d.default", (QueryProfileRegistry)null); - a2.set("e","a2.e.default", (QueryProfileRegistry)null); + a2.set("b","a2.b.default", null); + a2.set("c","a2.c.default", null); + a2.set("d","a2.d.default", null); + a2.set("e","a2.e.default", null); - profile.set("a",a1, (QueryProfileRegistry)null); // Must set profile references before overrides - profile.set("a.b","a.b.default-override", (QueryProfileRegistry)null); - profile.set("a.c","a.c.default-override", (QueryProfileRegistry)null); - profile.set("a.d","a.d.default-override", (QueryProfileRegistry)null); - profile.set("a.g","a.g.default-override", (QueryProfileRegistry)null); + profile.set("a",a1, null); // Must set profile references before overrides + profile.set("a.b","a.b.default-override", null); + profile.set("a.c","a.c.default-override", null); + profile.set("a.d","a.d.default-override", null); + profile.set("a.g","a.g.default-override", null); String[] d1=new String[] { "x1","y1" }; profile.set("a",a1,d1, null); @@ -181,6 +188,7 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("a.g.d3.runtime-override", d3RuntimeProfile.get("a.g", toMap("x=x2", "y=y1"))); } + @Test public void testVariantNotInBase() { QueryProfile test=new QueryProfile("test"); test.setDimensions(new String[] {"x"}); @@ -192,6 +200,7 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals(null,ctest.get("InX1Only")); } + @Test public void testVariantNotInBaseSpaceVariantValue() { QueryProfile test=new QueryProfile("test"); test.setDimensions(new String[] {"x"}); @@ -204,12 +213,13 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals(null,ctest.get("InX1Only")); } + @Test public void testDimensionsInSuperType() { QueryProfile parent=new QueryProfile("parent"); parent.setDimensions(new String[] {"x","y"}); QueryProfile child=new QueryProfile("child"); child.addInherited(parent); - child.set("a","a.default", (QueryProfileRegistry)null); + child.set("a","a.default", null); child.set("a","a.x1.y1",new String[] {"x1","y1"}, null); child.set("a","a.x1.y2",new String[] {"x1","y2"}, null); @@ -220,12 +230,13 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("a.x1.y2",cchild.get("a", toMap("x=x1","y=y2"))); } + @Test public void testDimensionsInSuperTypeRuntime() { QueryProfile parent=new QueryProfile("parent"); parent.setDimensions(new String[] {"x","y"}); QueryProfile child=new QueryProfile("child"); child.addInherited(parent); - child.set("a","a.default", (QueryProfileRegistry)null); + child.set("a","a.default", null); child.set("a", "a.x1.y1", new String[]{"x1", "y1"}, null); child.set("a", "a.x1.y2", new String[]{"x1", "y2"}, null); Properties overridable=new QueryProfileProperties(child.compile(null)); @@ -235,19 +246,20 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("a.x1.y2", overridable.get("a", toMap("x=x1", "y=y2"))); } + @Test public void testVariantsAreResolvedBeforeInheritance() { QueryProfile parent=new QueryProfile("parent"); parent.setDimensions(new String[] {"x","y"}); - parent.set("a","p.a.default", (QueryProfileRegistry)null); + parent.set("a","p.a.default", null); parent.set("a","p.a.x1.y1",new String[] {"x1","y1"}, null); parent.set("a","p.a.x1.y2",new String[] {"x1","y2"}, null); - parent.set("b","p.b.default", (QueryProfileRegistry)null); + parent.set("b","p.b.default", null); parent.set("b","p.b.x1.y1",new String[] {"x1","y1"}, null); parent.set("b","p.b.x1.y2",new String[] {"x1","y2"}, null); QueryProfile child=new QueryProfile("child"); child.setDimensions(new String[] {"x","y"}); child.addInherited(parent); - child.set("a","c.a.default", (QueryProfileRegistry)null); + child.set("a","c.a.default", null); child.set("a","c.a.x1.y1",new String[] {"x1","y1"}, null); CompiledQueryProfile cchild = child.compile(null); @@ -259,6 +271,7 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("p.b.x1.y2",cchild.get("b", toMap("x=x1", "y=y2"))); } + @Test public void testVariantsAreResolvedBeforeInheritanceSimplified() { QueryProfile parent=new QueryProfile("parent"); parent.setDimensions(new String[] {"x","y"}); @@ -267,26 +280,27 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { QueryProfile child=new QueryProfile("child"); child.setDimensions(new String[] {"x","y"}); child.addInherited(parent); - child.set("a","c.a.default", (QueryProfileRegistry)null); + child.set("a","c.a.default", null); assertEquals("c.a.default",child.compile(null).get("a", toMap("x=x1", "y=y2"))); } + @Test public void testVariantInheritance() { QueryProfile test=new QueryProfile("test"); test.setDimensions(new String[] {"x","y"}); QueryProfile defaultParent=new QueryProfile("defaultParent"); - defaultParent.set("a","a-default", (QueryProfileRegistry)null); + defaultParent.set("a","a-default", null); QueryProfile x1Parent=new QueryProfile("x1Parent"); - x1Parent.set("a","a-x1", (QueryProfileRegistry)null); - x1Parent.set("d","d-x1", (QueryProfileRegistry)null); - x1Parent.set("e","e-x1", (QueryProfileRegistry)null); + x1Parent.set("a","a-x1", null); + x1Parent.set("d","d-x1", null); + x1Parent.set("e","e-x1", null); QueryProfile x1y1Parent=new QueryProfile("x1y1Parent"); - x1y1Parent.set("a","a-x1y1", (QueryProfileRegistry)null); + x1y1Parent.set("a","a-x1y1", null); QueryProfile x1y2Parent=new QueryProfile("x1y2Parent"); - x1y2Parent.set("a","a-x1y2", (QueryProfileRegistry)null); - x1y2Parent.set("b","b-x1y2", (QueryProfileRegistry)null); - x1y2Parent.set("c","c-x1y2", (QueryProfileRegistry)null); + x1y2Parent.set("a","a-x1y2", null); + x1y2Parent.set("b","b-x1y2", null); + x1y2Parent.set("c","c-x1y2", null); test.addInherited(defaultParent); test.addInherited(x1Parent,new String[] {"x1"}); test.addInherited(x1y1Parent,new String[] {"x1","y1"}); @@ -323,11 +337,12 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("e-x1y2",ctest.get("e", toMap("x=x1", "y=y2"))); } + @Test public void testVariantInheritanceSimplified() { QueryProfile test=new QueryProfile("test"); test.setDimensions(new String[] {"x","y"}); QueryProfile x1y2Parent=new QueryProfile("x1y2Parent"); - x1y2Parent.set("c","c-x1y2", (QueryProfileRegistry)null); + x1y2Parent.set("c","c-x1y2", null); test.addInherited(x1y2Parent,new String[] {"x1","y2"}); test.set("c","c-x1",new String[] {"x1"}, null); @@ -339,13 +354,14 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("c-x1y2",ctest.get("c", toMap("x=x1", "y=y2"))); } + @Test public void testVariantInheritanceWithCompoundReferences() { QueryProfile test=new QueryProfile("test"); test.setDimensions(new String[] {"x"}); - test.set("a.b","default-a.b", (QueryProfileRegistry)null); + test.set("a.b","default-a.b", null); QueryProfile ac=new QueryProfile("ac"); - ac.set("a.c","referenced-a.c", (QueryProfileRegistry)null); + ac.set("a.c","referenced-a.c", null); test.addInherited(ac,new String[] {"x1"}); test.set("a.b","x1-a.b",new String[] {"x1"}, null); @@ -355,13 +371,14 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("Inherited variance reference overriding works","x1-a.b",ctest.get("a.b", toMap("x=x1"))); } + @Test public void testVariantInheritanceWithTwoLevelCompoundReferencesVariantAtFirstLevel() { QueryProfile test=new QueryProfile("test"); test.setDimensions(new String[] {"x"}); - test.set("o.a.b","default-a.b", (QueryProfileRegistry)null); + test.set("o.a.b","default-a.b", null); QueryProfile ac=new QueryProfile("ac"); - ac.set("o.a.c","referenced-a.c", (QueryProfileRegistry)null); + ac.set("o.a.c","referenced-a.c", null); test.addInherited(ac,new String[] {"x1"}); test.set("o.a.b","x1-a.b",new String[] {"x1"}, null); @@ -371,18 +388,19 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("Inherited variance reference overriding works","x1-a.b",ctest.get("o.a.b", toMap("x=x1"))); } + @Test public void testVariantInheritanceWithTwoLevelCompoundReferencesVariantAtSecondLevel() { QueryProfile test=new QueryProfile("test"); test.setDimensions(new String[] {"x"}); QueryProfile ac=new QueryProfile("ac"); - ac.set("a.c","referenced-a.c", (QueryProfileRegistry)null); + ac.set("a.c","referenced-a.c", null); test.addInherited(ac,new String[] {"x1"}); test.set("a.b","x1-a.b",new String[] {"x1"}, null); QueryProfile top=new QueryProfile("top"); - top.set("o.a.b","default-a.b", (QueryProfileRegistry)null); - top.set("o",test, (QueryProfileRegistry)null); + top.set("o.a.b","default-a.b", null); + top.set("o",test, null); CompiledQueryProfile ctop = top.compile(null); assertEquals("Basic functionality","default-a.b",ctop.get("o.a.b")); @@ -390,12 +408,13 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("Inherited variance reference does not override value set in referent","default-a.b",ctop.get("o.a.b", toMap("x=x1"))); // Note: Changed from x1-a.b in 4.2.3 } + @Test public void testVariantInheritanceOverridesBaseInheritance1() { QueryProfile test=new QueryProfile("test"); QueryProfile baseInherited=new QueryProfile("baseInherited"); - baseInherited.set("a.b","baseInherited-a.b", (QueryProfileRegistry)null); + baseInherited.set("a.b","baseInherited-a.b", null); QueryProfile variantInherited=new QueryProfile("variantInherited"); - variantInherited.set("a.b","variantInherited-a.b", (QueryProfileRegistry)null); + variantInherited.set("a.b","variantInherited-a.b", null); test.setDimensions(new String[] {"x"}); test.addInherited(baseInherited); test.addInherited(variantInherited,new String[] {"x1"}); @@ -405,12 +424,13 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("variantInherited-a.b",ctest.get("a.b",toMap("x=x1"))); } + @Test public void testVariantInheritanceOverridesBaseInheritance2() { QueryProfile test=new QueryProfile("test"); QueryProfile baseInherited=new QueryProfile("baseInherited"); - baseInherited.set("a.b","baseInherited-a.b", (QueryProfileRegistry)null); + baseInherited.set("a.b","baseInherited-a.b", null); QueryProfile variantInherited=new QueryProfile("variantInherited"); - variantInherited.set("a.b","variantInherited-a.b", (QueryProfileRegistry)null); + variantInherited.set("a.b","variantInherited-a.b", null); test.setDimensions(new String[] {"x"}); test.addInherited(baseInherited); test.addInherited(variantInherited,new String[] {"x1"}); @@ -422,22 +442,23 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("variant-a.c",ctest.get("a.c", toMap("x=x1"))); } + @Test public void testVariantInheritanceOverridesBaseInheritanceComplex() { QueryProfile defaultQP=new QueryProfile("default"); - defaultQP.set("model.defaultIndex","title", (QueryProfileRegistry)null); + defaultQP.set("model.defaultIndex","title", null); QueryProfile root=new QueryProfile("root"); root.addInherited(defaultQP); - root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + root.set("model.defaultIndex","default", null); QueryProfile querybest=new QueryProfile("querybest"); - querybest.set("defaultIndex","title", (QueryProfileRegistry)null); - querybest.set("queryString","best", (QueryProfileRegistry)null); + querybest.set("defaultIndex","title", null); + querybest.set("queryString","best", null); QueryProfile multi=new QueryProfile("multi"); multi.setDimensions(new String[] {"x"}); multi.addInherited(defaultQP); - multi.set("model",querybest, (QueryProfileRegistry)null); + multi.set("model",querybest, null); multi.addInherited(root,new String[] {"x1"}); multi.set("model.queryString","love",new String[] {"x1"}, null); @@ -452,26 +473,28 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("love",runtime.get("model.queryString", toMap("x=x1"))); } + @Test public void testVariantInheritanceOverridesBaseInheritanceComplexSimplified() { QueryProfile root=new QueryProfile("root"); - root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + root.set("model.defaultIndex","default", null); QueryProfile multi=new QueryProfile("multi"); multi.setDimensions(new String[] {"x"}); - multi.set("model.defaultIndex","title", (QueryProfileRegistry)null); + multi.set("model.defaultIndex","title", null); multi.addInherited(root,new String[] {"x1"}); assertEquals("default",multi.compile(null).get("model.defaultIndex", toMap("x=x1"))); } + @Test public void testVariantInheritanceOverridesBaseInheritanceMixed() { QueryProfile root=new QueryProfile("root"); - root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + root.set("model.defaultIndex","default", null); QueryProfile multi=new QueryProfile("multi"); multi.setDimensions(new String[] {"x"}); - multi.set("model.defaultIndex","title", (QueryProfileRegistry)null); - multi.set("model.queryString","modelQuery", (QueryProfileRegistry)null); + multi.set("model.defaultIndex","title", null); + multi.set("model.queryString","modelQuery", null); multi.addInherited(root,new String[] {"x1"}); multi.set("model.queryString","modelVariantQuery",new String[] {"x1"}, null); @@ -480,23 +503,24 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("modelVariantQuery",cmulti.get("model.queryString", toMap("x=x1"))); } + @Test public void testListVariantPropertiesNoCompounds() { QueryProfile parent1=new QueryProfile("parent1"); - parent1.set("a","parent1-a", (QueryProfileRegistry)null); // Defined everywhere - parent1.set("b","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants - parent1.set("c","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + parent1.set("a","parent1-a", null); // Defined everywhere + parent1.set("b","parent1-b", null); // Defined everywhere, but no variants + parent1.set("c","parent1-c", null); // Defined in both parents only QueryProfile parent2=new QueryProfile("parent2"); - parent2.set("a","parent2-a", (QueryProfileRegistry)null); - parent2.set("b","parent2-b", (QueryProfileRegistry)null); - parent2.set("c","parent2-c", (QueryProfileRegistry)null); - parent2.set("d","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + parent2.set("a","parent2-a", null); + parent2.set("b","parent2-b", null); + parent2.set("c","parent2-c", null); + parent2.set("d","parent2-d", null); // Defined in second parent only QueryProfile main=new QueryProfile("main"); main.setDimensions(new String[] {"x","y"}); main.addInherited(parent1); main.addInherited(parent2); - main.set("a","main-a", (QueryProfileRegistry)null); + main.set("a","main-a", null); main.set("a","main-a-x1",new String[] {"x1"}, null); main.set("e","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only main.set("f","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only @@ -504,19 +528,19 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { main.set("a","main-a-x1.y2",new String[] {"x1","y2"}, null); main.set("e","main-e-x1.y2",new String[] {"x1","y2"}, null); main.set("g","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only - main.set("b","main-b", (QueryProfileRegistry)null); + main.set("b","main-b", null); QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); - inheritedVariant1.set("a","inheritedVariant1-a", (QueryProfileRegistry)null); - inheritedVariant1.set("h","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant1.set("a","inheritedVariant1-a", null); + inheritedVariant1.set("h","inheritedVariant1-h", null); // Only defined in two inherited variants QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); - inheritedVariant2.set("a","inheritedVariant2-a", (QueryProfileRegistry)null); - inheritedVariant2.set("h","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants - inheritedVariant2.set("i","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + inheritedVariant2.set("a","inheritedVariant2-a", null); + inheritedVariant2.set("h","inheritedVariant2-h", null); // Only defined in two inherited variants + inheritedVariant2.set("i","inheritedVariant2-i", null); // Only defined in one inherited variant QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); - inheritedVariant3.set("j","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + inheritedVariant3.set("j","inheritedVariant3-j", null); // Only defined in one inherited variant, but inherited twice main.addInherited(inheritedVariant1,new String[] {"x1"}); main.addInherited(inheritedVariant3,new String[] {"x1"}); @@ -595,13 +619,14 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("parent2-d",listed.get("d")); } + @Test public void testListVariantPropertiesCompounds1Simplified() { QueryProfile main=new QueryProfile("main"); main.setDimensions(new String[] {"x","y"}); main.set("a.p1","main-a-x1",new String[] {"x1"}, null); QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); - inheritedVariant1.set("a.p1","inheritedVariant1-a", (QueryProfileRegistry)null); + inheritedVariant1.set("a.p1","inheritedVariant1-a", null); main.addInherited(inheritedVariant1,new String[] {"x1"}); Properties properties=new QueryProfileProperties(main.compile(null)); @@ -611,23 +636,24 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("main-a-x1",listed.get("a.p1")); } + @Test public void testListVariantPropertiesCompounds1() { QueryProfile parent1=new QueryProfile("parent1"); - parent1.set("a.p1","parent1-a", (QueryProfileRegistry)null); // Defined everywhere - parent1.set("b.p1","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants - parent1.set("c.p1","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + parent1.set("a.p1","parent1-a", null); // Defined everywhere + parent1.set("b.p1","parent1-b", null); // Defined everywhere, but no variants + parent1.set("c.p1","parent1-c", null); // Defined in both parents only QueryProfile parent2=new QueryProfile("parent2"); - parent2.set("a.p1","parent2-a", (QueryProfileRegistry)null); - parent2.set("b.p1","parent2-b", (QueryProfileRegistry)null); - parent2.set("c.p1","parent2-c", (QueryProfileRegistry)null); - parent2.set("d.p1","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + parent2.set("a.p1","parent2-a", null); + parent2.set("b.p1","parent2-b", null); + parent2.set("c.p1","parent2-c", null); + parent2.set("d.p1","parent2-d", null); // Defined in second parent only QueryProfile main=new QueryProfile("main"); main.setDimensions(new String[] {"x","y"}); main.addInherited(parent1); main.addInherited(parent2); - main.set("a.p1","main-a", (QueryProfileRegistry)null); + main.set("a.p1","main-a", null); main.set("a.p1","main-a-x1",new String[] {"x1"}, null); main.set("e.p1","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only main.set("f.p1","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only @@ -635,19 +661,19 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { main.set("a.p1","main-a-x1.y2",new String[] {"x1","y2"}, null); main.set("e.p1","main-e-x1.y2",new String[] {"x1","y2"}, null); main.set("g.p1","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only - main.set("b.p1","main-b", (QueryProfileRegistry)null); + main.set("b.p1","main-b", null); QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); - inheritedVariant1.set("a.p1","inheritedVariant1-a", (QueryProfileRegistry)null); - inheritedVariant1.set("h.p1","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant1.set("a.p1","inheritedVariant1-a", null); + inheritedVariant1.set("h.p1","inheritedVariant1-h", null); // Only defined in two inherited variants QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); - inheritedVariant2.set("a.p1","inheritedVariant2-a", (QueryProfileRegistry)null); - inheritedVariant2.set("h.p1","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants - inheritedVariant2.set("i.p1","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + inheritedVariant2.set("a.p1","inheritedVariant2-a", null); + inheritedVariant2.set("h.p1","inheritedVariant2-h", null); // Only defined in two inherited variants + inheritedVariant2.set("i.p1","inheritedVariant2-i", null); // Only defined in one inherited variant QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); - inheritedVariant3.set("j.p1","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + inheritedVariant3.set("j.p1","inheritedVariant3-j", null); // Only defined in one inherited variant, but inherited twice main.addInherited(inheritedVariant1,new String[] {"x1"}); main.addInherited(inheritedVariant3,new String[] {"x1"}); @@ -725,23 +751,24 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("parent2-d",listed.get("d.p1")); } + @Test public void testListVariantPropertiesCompounds2() { QueryProfile parent1=new QueryProfile("parent1"); - parent1.set("p1.a","parent1-a", (QueryProfileRegistry)null); // Defined everywhere - parent1.set("p1.b","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants - parent1.set("p1.c","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + parent1.set("p1.a","parent1-a", null); // Defined everywhere + parent1.set("p1.b","parent1-b", null); // Defined everywhere, but no variants + parent1.set("p1.c","parent1-c", null); // Defined in both parents only QueryProfile parent2=new QueryProfile("parent2"); - parent2.set("p1.a","parent2-a", (QueryProfileRegistry)null); - parent2.set("p1.b","parent2-b", (QueryProfileRegistry)null); - parent2.set("p1.c","parent2-c", (QueryProfileRegistry)null); - parent2.set("p1.d","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + parent2.set("p1.a","parent2-a", null); + parent2.set("p1.b","parent2-b", null); + parent2.set("p1.c","parent2-c", null); + parent2.set("p1.d","parent2-d", null); // Defined in second parent only QueryProfile main=new QueryProfile("main"); main.setDimensions(new String[] {"x","y"}); main.addInherited(parent1); main.addInherited(parent2); - main.set("p1.a","main-a", (QueryProfileRegistry)null); + main.set("p1.a","main-a", null); main.set("p1.a","main-a-x1",new String[] {"x1"}, null); main.set("p1.e","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only main.set("p1.f","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only @@ -749,19 +776,19 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { main.set("p1.a","main-a-x1.y2",new String[] {"x1","y2"}, null); main.set("p1.e","main-e-x1.y2",new String[] {"x1","y2"}, null); main.set("p1.g","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only - main.set("p1.b","main-b", (QueryProfileRegistry)null); + main.set("p1.b","main-b", null); QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); - inheritedVariant1.set("p1.a","inheritedVariant1-a", (QueryProfileRegistry)null); - inheritedVariant1.set("p1.h","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant1.set("p1.a","inheritedVariant1-a", null); + inheritedVariant1.set("p1.h","inheritedVariant1-h", null); // Only defined in two inherited variants QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); - inheritedVariant2.set("p1.a","inheritedVariant2-a", (QueryProfileRegistry)null); - inheritedVariant2.set("p1.h","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants - inheritedVariant2.set("p1.i","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + inheritedVariant2.set("p1.a","inheritedVariant2-a", null); + inheritedVariant2.set("p1.h","inheritedVariant2-h", null); // Only defined in two inherited variants + inheritedVariant2.set("p1.i","inheritedVariant2-i", null); // Only defined in one inherited variant QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); - inheritedVariant3.set("p1.j","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + inheritedVariant3.set("p1.j","inheritedVariant3-j", null); // Only defined in one inherited variant, but inherited twice main.addInherited(inheritedVariant1,new String[] {"x1"}); main.addInherited(inheritedVariant3,new String[] {"x1"}); @@ -839,17 +866,18 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("parent2-d",listed.get("p1.d")); } + @Test public void testQueryProfileReferences() { QueryProfile main=new QueryProfile("main"); main.setDimensions(new String[] {"x1"}); QueryProfile referencedMain=new QueryProfile("referencedMain"); - referencedMain.set("r1","mainReferenced-r1", (QueryProfileRegistry)null); // In both - referencedMain.set("r2","mainReferenced-r2", (QueryProfileRegistry)null); // Only in this + referencedMain.set("r1","mainReferenced-r1", null); // In both + referencedMain.set("r2","mainReferenced-r2", null); // Only in this QueryProfile referencedVariant=new QueryProfile("referencedVariant"); - referencedVariant.set("r1","variantReferenced-r1", (QueryProfileRegistry)null); // In both - referencedVariant.set("r3","variantReferenced-r3", (QueryProfileRegistry)null); // Only in this + referencedVariant.set("r1","variantReferenced-r1", null); // In both + referencedVariant.set("r3","variantReferenced-r3", null); // Only in this - main.set("a",referencedMain, (QueryProfileRegistry)null); + main.set("a",referencedMain, null); main.set("a",referencedVariant,new String[] {"x1"}, null); Properties properties=new QueryProfileProperties(main.compile(null)); @@ -868,19 +896,20 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("variantReferenced-r3",listed.get("a.r3")); } + @Test public void testQueryProfileReferencesWithSubstitution() { QueryProfile main=new QueryProfile("main"); main.setDimensions(new String[] {"x1"}); QueryProfile referencedMain=new QueryProfile("referencedMain"); - referencedMain.set("r1","%{prefix}mainReferenced-r1", (QueryProfileRegistry)null); // In both - referencedMain.set("r2","%{prefix}mainReferenced-r2", (QueryProfileRegistry)null); // Only in this + referencedMain.set("r1","%{prefix}mainReferenced-r1", null); // In both + referencedMain.set("r2","%{prefix}mainReferenced-r2", null); // Only in this QueryProfile referencedVariant=new QueryProfile("referencedVariant"); - referencedVariant.set("r1","%{prefix}variantReferenced-r1", (QueryProfileRegistry)null); // In both - referencedVariant.set("r3","%{prefix}variantReferenced-r3", (QueryProfileRegistry)null); // Only in this + referencedVariant.set("r1","%{prefix}variantReferenced-r1", null); // In both + referencedVariant.set("r3","%{prefix}variantReferenced-r3", null); // Only in this - main.set("a",referencedMain, (QueryProfileRegistry)null); + main.set("a",referencedMain, null); main.set("a",referencedVariant,new String[] {"x1"}, null); - main.set("prefix","mainPrefix:", (QueryProfileRegistry)null); + main.set("prefix","mainPrefix:", null); main.set("prefix","variantPrefix:",new String[] {"x1"}, null); Properties properties=new QueryProfileProperties(main.compile(null)); @@ -899,11 +928,12 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("variantPrefix:variantReferenced-r3",listed.get("a.r3")); } + @Test public void testNewsCase1() { QueryProfile shortcuts=new QueryProfile("shortcuts"); shortcuts.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); - shortcuts.set("testout","outside", (QueryProfileRegistry)null); - shortcuts.set("test.out","dotoutside", (QueryProfileRegistry)null); + shortcuts.set("testout","outside", null); + shortcuts.set("test.out","dotoutside", null); shortcuts.set("testin","inside",new String[] {"yahoo","ca","sc"}, null); shortcuts.set("test.in","dotinside",new String[] {"yahoo","ca","sc"}, null); @@ -920,6 +950,7 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("dotinside",query.properties().get("test.in")); } + @Test public void testNewsCase2() { QueryProfile test=new QueryProfile("test"); test.setDimensions("sort,resulttypes,rss,age,intl,testid".split(",")); @@ -945,6 +976,7 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("10",sourceValues.get("count")); } + @Test public void testRuntimeAssignmentInClone() { QueryProfile test=new QueryProfile("test"); test.setDimensions(new String[] {"x"}); @@ -977,12 +1009,13 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { assertEquals("51",qMain.properties().get("a.b",x1m)); } + @Test public void testIncompatibleDimensions() { QueryProfile alert = new QueryProfile("alert"); QueryProfile backendBase = new QueryProfile("backendBase"); backendBase.setDimensions(new String[] { "sort", "resulttypes", "rss" }); - backendBase.set("custid", "s", (QueryProfileRegistry)null); + backendBase.set("custid", "s", null); QueryProfile backend = new QueryProfile("backend"); backend.setDimensions(new String[] { "sort", "offset", "resulttypes", "rss", "age", "lang", "fr", "entry" }); @@ -992,19 +1025,20 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { web.setDimensions(new String[] { "entry", "recency" }); web.set("fr", "alerts", new String[] { "alert" }, null); - alert.set("config.backend.vertical.news", backend, (QueryProfileRegistry)null); - alert.set("config.backend.multimedia", web, (QueryProfileRegistry)null); + alert.set("config.backend.vertical.news", backend, null); + alert.set("config.backend.multimedia", web, null); backend.set("custid", "yahoo/alerts", new String[] { null, null, null, null, null, "en-US", null, "alert"}, null); CompiledQueryProfile cAlert = alert.compile(null); assertEquals("yahoo/alerts", cAlert.get("config.backend.vertical.news.custid", toMap("entry=alert", "intl=us", "lang=en-US"))); } + @Test public void testIncompatibleDimensionsSimplified() { QueryProfile alert = new QueryProfile("alert"); QueryProfile backendBase = new QueryProfile("backendBase"); - backendBase.set("custid", "s", (QueryProfileRegistry)null); + backendBase.set("custid", "s", null); QueryProfile backend = new QueryProfile("backend"); backend.setDimensions(new String[] { "sort", "lang", "fr", "entry" }); @@ -1015,8 +1049,8 @@ public class QueryProfileVariantsTestCase extends junit.framework.TestCase { web.setDimensions(new String[] { "entry", "recency" }); web.set("fr", "alerts", new String[] { "alert" }, null); - alert.set("vertical", backend, (QueryProfileRegistry)null); - alert.set("multimedia", web, (QueryProfileRegistry)null); + alert.set("vertical", backend, null); + alert.set("multimedia", web, null); CompiledQueryProfile cAlert = alert.compile(null); assertEquals("yahoo/alerts", cAlert.get("vertical.custid", toMap("entry=alert", "intl=us", "lang=en-US"))); diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index ae2e479e8bf..b03dd66c817 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -20,7 +20,6 @@ vespa_define_module( src/vespa/documentapi/messagebus src/vespa/documentapi/messagebus/messages src/vespa/documentapi/messagebus/policies - src/vespa/documentapi/messagebus/systemstate TEST_DEPENDS messagebus_messagebus-test @@ -34,5 +33,4 @@ vespa_define_module( src/tests/priority src/tests/replymerger src/tests/routablefactory - src/tests/systemstate ) diff --git a/documentapi/src/testlist.txt b/documentapi/src/testlist.txt index 851741bd24a..270cd72f524 100644 --- a/documentapi/src/testlist.txt +++ b/documentapi/src/testlist.txt @@ -4,6 +4,5 @@ tests/policies tests/policyfactory tests/priority tests/routablefactory -tests/systemstate tests/loadtypes tests/replymerger diff --git a/documentapi/src/tests/policies/testframe.cpp b/documentapi/src/tests/policies/testframe.cpp index 024a0c2fdaa..d99d99b9ec0 100644 --- a/documentapi/src/tests/policies/testframe.cpp +++ b/documentapi/src/tests/policies/testframe.cpp @@ -322,13 +322,6 @@ TestFrame::waitSlobrok(const string &pattern, uint32_t cnt) return false; } -SystemStateHandle -TestFrame::getSystemState() -{ - mbus::IProtocol * protocol = _mbus->getProtocol(DocumentProtocol::NAME); - return SystemStateHandle(static_cast<DocumentProtocol&>(*protocol).getSystemState()); -} - void TestFrame::handleReply(mbus::Reply::UP reply) { diff --git a/documentapi/src/tests/policies/testframe.h b/documentapi/src/tests/policies/testframe.h index 15ec61aff7b..cc5101905a1 100644 --- a/documentapi/src/tests/policies/testframe.h +++ b/documentapi/src/tests/policies/testframe.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/network/identity.h> #include <vespa/messagebus/network/inetwork.h> @@ -190,12 +189,6 @@ public: */ mbus::Receptor &getReceptor() { return _handler; } - /** - * Returns the system state from contained document protocol. - * - * @return Handle to the system state. - */ - documentapi::SystemStateHandle getSystemState(); void handleReply(mbus::Reply::UP reply) override; }; diff --git a/documentapi/src/tests/systemstate/.gitignore b/documentapi/src/tests/systemstate/.gitignore deleted file mode 100644 index 3f52bc38742..00000000000 --- a/documentapi/src/tests/systemstate/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -systemstate_test -documentapi_systemstate_test_app diff --git a/documentapi/src/tests/systemstate/CMakeLists.txt b/documentapi/src/tests/systemstate/CMakeLists.txt deleted file mode 100644 index b275f2b2fd0..00000000000 --- a/documentapi/src/tests/systemstate/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(documentapi_systemstate_test_app TEST - SOURCES - systemstate.cpp - DEPENDS - documentapi -) -vespa_add_test(NAME documentapi_systemstate_test_app COMMAND documentapi_systemstate_test_app) diff --git a/documentapi/src/tests/systemstate/DESC b/documentapi/src/tests/systemstate/DESC deleted file mode 100644 index 19dbc9195f1..00000000000 --- a/documentapi/src/tests/systemstate/DESC +++ /dev/null @@ -1,3 +0,0 @@ -This is a unit test for the system state parser and the corresponding NodeState class. It mirrors the -StateParserTestCase available in the java implementation of message bus. It consists of tests that verify that parsing -works, pathing works, encoding/decoding works, and finally that the NodeState class works as intended. diff --git a/documentapi/src/tests/systemstate/FILES b/documentapi/src/tests/systemstate/FILES deleted file mode 100644 index e1d0e026d31..00000000000 --- a/documentapi/src/tests/systemstate/FILES +++ /dev/null @@ -1 +0,0 @@ -systemstate.cpp diff --git a/documentapi/src/tests/systemstate/systemstate.cpp b/documentapi/src/tests/systemstate/systemstate.cpp deleted file mode 100644 index 3e42c94950a..00000000000 --- a/documentapi/src/tests/systemstate/systemstate.cpp +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/documentapi/messagebus/systemstate/systemstate.h> -#include <vespa/documentapi/messagebus/systemstate/nodestate.h> -#include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/log/log.h> -LOG_SETUP("systemstate_test"); - -using namespace documentapi; - -class Test : public vespalib::TestApp { -public: - int Main() override; - void testParser(); - void testPathing(); - void testState(); - void testEncoding(); - void testHandle(); - void testCompact(); - -private: - void assertParser(const string &state, const string &expected = ""); -}; - -TEST_APPHOOK(Test); - -int -Test::Main() -{ - TEST_INIT("systemstate_test"); - - testParser(); TEST_FLUSH(); - testPathing(); TEST_FLUSH(); - testState(); TEST_FLUSH(); - testEncoding(); TEST_FLUSH(); - testHandle(); TEST_FLUSH(); - testCompact(); TEST_FLUSH(); - - TEST_DONE(); - return 0; -} - -void -Test::testParser() -{ - assertParser("storage"); - assertParser("storage?", "ERROR"); - assertParser("storage?a", "ERROR"); - assertParser("storage?a=", "ERROR"); - assertParser("storage?a=1"); - assertParser("storage?a=1&", "ERROR"); - assertParser("storage?a=1&b", "ERROR"); - assertParser("storage?a=1&b=2"); - assertParser("storage?a=1&b=2 search"); - assertParser("storage?a=1&b=2 search?", "ERROR"); - assertParser("storage?a=1&b=2 search?a", "ERROR"); - assertParser("storage?a=1&b=2 search?a=", "ERROR"); - assertParser("storage?a=1&b=2 search?a=1"); - assertParser("storage?a=1&b=2 search?a=1&", "ERROR"); - assertParser("storage?a=1&b=2 search?a=1&b", "ERROR"); - assertParser("storage?a=1&b=2 search?a=1&b=", "ERROR"); - assertParser("storage?a=1&b=2 search?a=1&b=2"); - - assertParser("storage"); - assertParser("storage/"); - assertParser("storage/?", "ERROR"); - assertParser("storage/?a", "ERROR"); - assertParser("storage/?a=", "ERROR"); - assertParser("storage/?a=1"); - assertParser("storage/cluster.storage"); - assertParser("storage/cluster.storage/"); - - assertParser("storage?a=1"); - assertParser("storage/?a=1"); - assertParser("storage/.?a=1"); - assertParser("storage/./?a=1"); - assertParser("storage/./cluster.storage?a=1"); - assertParser("storage/./cluster.storage/?a=1"); - assertParser("storage/./cluster.storage/..?a=1"); - assertParser("storage/./cluster.storage/../?a=1"); - assertParser("storage/./cluster.storage/../storage?a=1"); - assertParser("storage/./cluster.storage/../storage/?a=1"); -} - -void -Test::testPathing() -{ - assertParser("storage?a=1", "storage?a=1"); - assertParser("storage/?a=1", "storage?a=1"); - assertParser("storage/.?a=1", "storage?a=1"); - assertParser("storage/./?a=1", "storage?a=1"); - assertParser("storage/./cluster.storage?a=1", "storage/cluster.storage?a=1"); - assertParser("storage/./cluster.storage/?a=1", "storage/cluster.storage?a=1"); - assertParser("storage/./cluster.storage/..?a=1", "storage?a=1"); - assertParser("storage/./cluster.storage/../?a=1", "storage?a=1"); - assertParser("storage/./cluster.storage/../storage?a=1", "storage/storage?a=1"); - assertParser("storage/./cluster.storage/../storage/?a=1", "storage/storage?a=1"); - - assertParser("a?p1=1 a/b?p2=2 a/b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - assertParser("a .?p1=1 ./b?p2=2 ./b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - assertParser("a .?p1=1 ./../a/b/ .?p2=2 c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - assertParser("a/./ .?p1=1 ../a/b/c/.. .?p2=2 ./c/../c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - assertParser("a/b/c/d/ ../../ ../ ../a .?p1=1 ./b?p2=2 ./ ../a/b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); - - assertParser("a/b/c/d?p1=1 a?p2=2", "a?p2=2 a/b/c/d?p1=1"); - assertParser("a/b/c/d/?p1=1 /a?p2=2", "a?p2=2 a/b/c/d?p1=1"); - assertParser("/a/b/c/d/?p1=1 /a?p2=2", "a?p2=2 a/b/c/d?p1=1"); - - assertParser("a .?p1=1", "a?p1=1"); - assertParser("a/b .?p1=1", "a/b?p1=1"); - assertParser("a/b c?p1=1 d?p2=2", "a/b/c?p1=1 a/b/d?p2=2"); -} - -void -Test::testState() -{ - NodeState state; - state - .addChild("distributor", NodeState() - .setState("n", "27")) - .addChild("storage", NodeState() - .setState("n", "170") - .addChild("2", NodeState() - .setState("s", "d")) - .addChild("13", NodeState() - .setState("s", "r") - .setState("c", "0.0"))); - - EXPECT_EQUAL("27", state.getState("distributor/n")); - EXPECT_EQUAL("170", state.getState("storage/n")); - EXPECT_EQUAL("d", state.getState("storage/2/s")); - EXPECT_EQUAL("r", state.getState("storage/13/s")); - EXPECT_EQUAL("0.0", state.getState("storage/13/c")); - - EXPECT_EQUAL("27", state.getChild("distributor")->getState("n")); - EXPECT_EQUAL("170", state.getChild("storage")->getState("n")); - EXPECT_EQUAL("d", state.getChild("storage")->getChild("2")->getState("s")); - EXPECT_EQUAL("r", state.getChild("storage")->getChild("13")->getState("s")); - EXPECT_EQUAL("0.0", state.getChild("storage")->getChild("13")->getState("c")); -} - -void -Test::testEncoding() -{ - NodeState state; - state.setState("foo", "http://search.yahoo.com/?query=bar"); - LOG(info, "'%s'", state.toString().c_str()); - EXPECT_EQUAL(".?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); - assertParser(state.toString(), state.toString()); - - state = NodeState() - .addChild("foo:bar", NodeState() - .setState("foo", "http://search.yahoo.com/?query=bar")); - LOG(info, "'%s'", state.toString().c_str()); - EXPECT_EQUAL("foo%3Abar?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); - assertParser(state.toString(), state.toString()); - - state = NodeState() - .addChild("foo/bar", NodeState() - .setState("foo", "http://search.yahoo.com/?query=bar")); - LOG(info, "'%s'", state.toString().c_str()); - EXPECT_EQUAL("foo/bar?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); - assertParser(state.toString(), state.toString()); -} - -void -Test::testHandle() -{ - SystemState::UP state(SystemState::newInstance("")); - ASSERT_TRUE(state.get() != NULL); - - SystemStateHandle handle(*state); - ASSERT_TRUE(handle.isValid()); - - SystemStateHandle hoe(std::move(handle)); - ASSERT_TRUE(!handle.isValid()); - ASSERT_TRUE(hoe.isValid()); -} - -void -Test::testCompact() -{ - NodeState state; - state - .setState("a/b0/s", "d") - .setState("a/b0/c0/s", "d") - .setState("a/b0/c1/s", "d") - .setState("a/b1/s", "d") - .setState("a/b1/c0/s", "d") - .setState("a/b1/c1/s", "d"); - EXPECT_EQUAL("a/b0?s=d a/b0/c0?s=d a/b0/c1?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); - - state.removeChild("a/b0/c0"); - EXPECT_EQUAL("a/b0?s=d a/b0/c1?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); - - state.removeState("a/b0/c1/s"); - EXPECT_EQUAL("a/b0?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); - - state.setState("a/b1/c0/s", ""); - EXPECT_EQUAL("a/b0?s=d a/b1?s=d a/b1/c1?s=d", state.toString()); - - state.removeChild("a/b1"); - EXPECT_EQUAL("a/b0?s=d", state.toString()); - - state.removeChild("a"); - EXPECT_EQUAL("", state.toString()); -} - -void -Test::assertParser(const string &state, const string &expected) -{ - SystemState::UP obj = SystemState::newInstance(state); - if (obj.get() == NULL) { - EXPECT_EQUAL("ERROR", expected); - } - else { - SystemStateHandle handle(*obj); - LOG(info, "'%s' => '%s'", state.c_str(), handle.getRoot().toString().c_str()); - if (!expected.empty()) { - EXPECT_EQUAL(expected, handle.getRoot().toString()); - } - } -} - diff --git a/documentapi/src/vespa/documentapi/CMakeLists.txt b/documentapi/src/vespa/documentapi/CMakeLists.txt index 42998182f56..46bc0239834 100644 --- a/documentapi/src/vespa/documentapi/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_add_library(documentapi $<TARGET_OBJECTS:documentapi_documentapimessagebus> $<TARGET_OBJECTS:documentapi_documentapimessages> $<TARGET_OBJECTS:documentapi_documentapipolicies> - $<TARGET_OBJECTS:documentapi_documentapisystemstate> $<TARGET_OBJECTS:documentapi_documentapiloadtypes> INSTALL lib64 DEPENDS diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index d92fbdfb941..d2661d0fe6c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -28,7 +28,6 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, const string &configId) : _routingPolicyRepository(new RoutingPolicyRepository()), _routableRepository(new RoutableRepository(loadTypes)), - _systemState(SystemState::newInstance("")), _repo(repo) { // Prepare config string for routing policy factories. diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index 0a6c10ace77..32e6dc1d95e 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/documentapi/messagebus/systemstate/systemstate.h> #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/iprotocol.h> #include <vespa/messagebus/reply.h> #include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/documentapi/common.h> namespace vespalib { class VersionSpecification; @@ -28,7 +28,6 @@ class DocumentProtocol final : public mbus::IProtocol { private: std::unique_ptr<RoutingPolicyRepository> _routingPolicyRepository; std::unique_ptr<RoutableRepository> _routableRepository; - std::unique_ptr<SystemState> _systemState; std::shared_ptr<const document::DocumentTypeRepo> _repo; public: @@ -295,13 +294,6 @@ public: */ static bool hasOnlyErrorsOfType(const mbus::Reply &reply, uint32_t errCode); - /** - * Returns the curren state of the system, as observed by this protocol. This state object may be freely - * modified by the caller. - * - * @return The system state. - */ - SystemState &getSystemState() { return *_systemState; } const mbus::string &getName() const override { return NAME; } mbus::IRoutingPolicy::UP createPolicy(const mbus::string &name, const mbus::string ¶m) const override; mbus::Blob encode(const vespalib::Version &version, const mbus::Routable &routable) const override; diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h index 05202ad0de9..e2bf5119c58 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h @@ -2,7 +2,6 @@ #pragma once #include "iroutingpolicyfactory.h" -#include <vespa/documentapi/messagebus/systemstate/systemstate.h> namespace document { class DocumentTypeRepo; } diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore b/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore deleted file mode 100644 index 5dae353d999..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.depend -Makefile diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt deleted file mode 100644 index 45ca04e2cb3..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(documentapi_documentapisystemstate OBJECT - SOURCES - nodestate.cpp - systemstate.cpp - systemstatehandle.cpp - urlencoder.cpp - DEPENDS -) diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp deleted file mode 100644 index 42c573ef09e..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "nodestate.h" -#include "urlencoder.h" - -#include <vespa/log/log.h> -LOG_SETUP(".nodestate"); - -using namespace documentapi; - -NodeState::NodeState() : - _parent(NULL), - _id(""), - _children(), - _state() -{ - // empty -} - -NodeState::NodeState(const NodeState &rhs) : - _parent(rhs._parent), - _id(rhs._id), - _children(rhs._children), - _state(rhs._state) -{ - // empty -} - -NodeState::NodeState(StateMap args) : - _parent(NULL), - _id(""), - _children(), - _state(args) -{ - // empty -} - -NodeState::~NodeState() { } - -NodeState & -NodeState::addChild(const string &key, const NodeState &child) -{ - getChild(key, true)->copy(child); - return *this; -} - -NodeState * -NodeState::getChild(const string &key, bool force) -{ - if (key.empty()) { - return this; - } - - // Find first not-self location item. - size_t from = 0, to = key.find('/'); - while (to != string::npos && key.substr(from, to - from) == ".") { - from = to + 1; - to = key.find('/', from); - } - string arr0 = to != string::npos ? key.substr(from, to - from) : key.substr(from); - string arr1 = to != string::npos ? key.substr(to + 1) : ""; - - // Reference this or parent. - if (arr0 == ".") { - return this; - } - if (arr0 == "..") { - if (_parent == NULL) { - LOG(error, "Location string '%s' requests a parent above the top-most node, returning self to avoid crash.", - key.c_str()); - return this; - } - return _parent->getChild(arr1, force); - } - - // Look for child, forcing it if requested. - ChildMap::iterator it = _children.find(arr0); - if (it == _children.end()) { - if (!force) { - return NULL; - } - _children[arr0] = NodeState::SP(new NodeState()); - _children[arr0]->setParent(*this, arr0); - } - if (to != string::npos) { - return _children[arr0]->getChild(arr1, force); - } - return _children[arr0].get(); -} - -const NodeState::ChildMap & -NodeState::getChildren() const -{ - return _children; -} - -NodeState & -NodeState::removeChild(const string &key) -{ - if (key.empty()) { - return *this; - } - size_t pos = key.find_last_of('/'); - if (pos != string::npos) { - NodeState* parent = getChild(key.substr(0, pos), false); - if (parent != NULL) { - return parent->removeChild(key.substr(pos + 1)); - } - } - else { - _children.erase(key); - } - return compact(); -} - -const string -NodeState::getState(const string &key) -{ - if (key.empty()) { - return ""; - } - size_t pos = key.find_last_of('/'); - if (pos != string::npos) { - NodeState* parent = getChild(key.substr(0, pos), false); - return parent != NULL ? parent->getState(key.substr(pos + 1)) : ""; - } - StateMap::iterator it = _state.find(key); - return it != _state.end() ? it->second : ""; -} - -NodeState & -NodeState::setState(const string &key, const string &value) -{ - if (key.empty()) { - return *this; - } - size_t pos = key.find_last_of('/'); - if (pos != string::npos) { - getChild(key.substr(0, pos), true)->setState(key.substr(pos + 1), value); - } - else { - if (value.empty()) { - return removeState(key); - } - else { - _state[key] = value; - } - } - return *this; -} - -NodeState & -NodeState::removeState(const string &key) -{ - if (key.empty()) { - return *this; - } - size_t pos = key.find_last_of('/'); - if (pos != string::npos) { - NodeState* parent = getChild(key.substr(0, pos), false); - if (parent != NULL) { - return parent->removeState(key.substr(pos + 1)); - } - } - else { - _state.erase(key); - } - return compact(); -} - -NodeState & -NodeState::compact() -{ - if (_state.empty() && _children.empty()) { - if (_parent != NULL) { - return _parent->removeChild(_id); - } - } - return *this; -} - -NodeState & -NodeState::copy(const NodeState &node) -{ - for (StateMap::const_iterator it = node._state.begin(); - it != node._state.end(); ++it) { - _state[it->first] = it->second; - } - for (ChildMap::const_iterator it = node._children.begin(); - it != node._children.end(); ++it) { - getChild(it->first, true)->copy(*it->second); - } - return *this; -} - -NodeState & -NodeState::clear() -{ - _state.clear(); - _children.clear(); - return compact(); -} - -NodeState & -NodeState::setParent(NodeState &parent, const string &id) -{ - _parent = &parent; - _id = id; - return *this; -} - -const string -NodeState::toString() const -{ - const std::string ret = toString(""); - size_t pos = ret.find_last_not_of(' '); - return pos != string::npos ? ret.substr(0, pos + 1) : ret; -} - -const string -NodeState::toString(const string &prefix) const -{ - string ret; - if (!_state.empty()) { - string str; - for (StateMap::const_iterator it = _state.begin(); - it != _state.end(); ++it) { - str += it->first + "=" + URLEncoder::encode(it->second) + "&"; - } - ret += (prefix.empty() ? ".?" : prefix + "?") + str.substr(0, str.size() - 1) + " "; - } - string pre = prefix.empty() ? "" : (prefix + "/"); - for (ChildMap::const_iterator it = _children.begin(); - it != _children.end(); ++it) { - ret += it->second->toString(pre + URLEncoder::encode(it->first)); - } - return ret; -} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h deleted file mode 100644 index eff684f5bfe..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/documentapi/common.h> -#include <map> - -namespace documentapi { - -/** - * A node state is a single node in an annotatet tree of such nodes. It contains a reference to its parent - * node, a list of named child nodes, as well as a mapping of (key, value) pairs that constitute the annotated - * state of this node. To create an instance of a node state tree, one can either use the {@link SystemState} - * factory class, or one can programmatically construct each node and use the chaining capabilities of its - * set-methods to compact the necessary code. - */ -class NodeState { -public: - typedef std::unique_ptr<NodeState> UP; - typedef std::shared_ptr<NodeState> SP; - typedef std::map<string, string> StateMap; - typedef std::map<string, NodeState::SP> ChildMap; - -private: - NodeState* _parent; - string _id; - ChildMap _children; - StateMap _state; - - /** - * Compacts the system state tree from this node upwards. This will delete itself if it has a parent, but - * no internal state and no children. - * - * @return This or the first non-null ancestor, to allow chaining. - */ - NodeState &compact(); - - /** - * Returns a string representation of this node state. - * - * @param prefix The prefix to use for this string. - * @return A string representation of this. - */ - const string toString(const string &prefix) const; - -public: - NodeState(NodeState && rhs) = default; - NodeState & operator = (NodeState && rhs) = default; - NodeState & operator = (const NodeState & rhs) = default; - /** - * Creates a node state that no internal content. - */ - NodeState(); - - /** - * Creates a node state as a copy of another. - * - * @param rhs The state to copy. - */ - NodeState(const NodeState &rhs); - - /** - * Creates a node state based on a list of argument objects. These arguments are iterated and added to - * this node's internal state map. - * - * @param args The arguments to use as state. - */ - NodeState(StateMap args); - - ~NodeState(); - - /** - * Adds a child to this node at the given location. The key can be a location string, in which case the - * necessary intermediate node states are created. - * - * @param key The location at which to add the child. - * @param child The child node to add. - * @return This, to allow chaining. - */ - NodeState &addChild(const string &key, const NodeState &child); - - /** - * Returns the child at the given location relative to this. This method can be forced to return a child - * node even if it does not exist, by adding all intermediate nodes and the target node itself. - * - * @param key The location of the child to return. - * @param force Whether or not to force a return value by creating missing nodes. - * @return The child object, null if not found. - */ - NodeState *getChild(const string &key, bool force = false); - - /** - * Returns the map of child nodes for iteration. - * - * @return The internal child map. - */ - const ChildMap &getChildren() const; - - /** - * Removes the named child node from this node, and attempts to compact the system state from this node - * upwards by removing empty nodes. - * - * @param key The child to remove. - * @return The result of invoking {@link #compact} after the remove. - */ - NodeState &removeChild(const string &key); - - /** - * Retrieves some arbitrary state information for a given key. The key can be a location string, in which - * case the necessary intermediate nodes are traversed. If the key is not found, this method returns - * null. This method can not be const because it uses the non-const method {@link #getChild} to resolve a - * pathed key. - * - * @param key The name of the state information to return. - * @return The value of the state key. - */ - const string getState(const string &key); - - /** - * Sets some arbitrary state data in this node. The key can be a location string, in which case the - * necessary intermediate nodes are traversed and even created if missing. - * - * @param key The key to set. - * @param value The value to assign to the key. - * @return This, to allow chaining. - */ - NodeState &setState(const string &key, const string &value); - - /** - * Removes the named (key, value) state pair from this node, and attempts to compact the system state from - * this node upwards by removing empty nodes. - * - * @param key The state variable to clear. - * @return The result of invoking {@link #compact} after the remove. - */ - NodeState &removeState(const string &key); - - /** - * Copies the state content of another node state object into this. - * - * @param node The node state to copy into this. - * @return This, to allow chaining. - */ - NodeState ©(const NodeState &node); - - /** - * Clears both the internal state and child list, then compacts the tree from this node upwards. - * - * @return The result of invoking {@link #compact} after the remove. - */ - NodeState &clear(); - - /** - * Sets the parent of this node. - * - * @param parent The parent node. - * @param id The identifier of this node as seen in the parent. - * @return This, to allow chaining. - */ - NodeState &setParent(NodeState &parent, const string &id); - - /** - * Returns a string representation of this node state. - * - * @return A string representation of this. - */ - const string toString() const; -}; - -} - diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp deleted file mode 100644 index 0556e859c42..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "systemstate.h" -#include "nodestate.h" -#include <vespa/vespalib/util/sync.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <boost/spirit/include/classic_core.hpp> -#include <boost/spirit/include/classic_parse_tree.hpp> -#include <boost/spirit/include/classic_tree_to_xml.hpp> -#include <boost/spirit/include/classic_chset.hpp> -#include <boost/spirit/include/classic_escape_char.hpp> -#include <boost/spirit/include/classic_grammar_def.hpp> - -#include <vespa/log/log.h> -LOG_SETUP(".systemstate"); - -using namespace documentapi; - -/** - * This class implements a boost::spirit type parser for the system state string. All contained names - * confirm to the boost::spirit naming convention, and care should therefore be taken if one wishes - * to modify any of these. Note that all content is inlined, just as all of boost::spirit, so this is - * therefore contained in the .cpp file instead of a separate .h file. - */ -struct SystemStateGrammar : public boost::spirit::classic::grammar<SystemStateGrammar> { - enum RuleId { - id_hexChar = 1, - id_hexCode, - id_alphaNum, - id_string, - id_argument, - id_argumentList, - id_locationItem, - id_location, - id_systemState - }; - - template <typename Scanner> - struct gram_base { - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_hexChar> > rule_hexChar; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_hexCode> > rule_hexCode; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_alphaNum> > rule_alphaNum; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_string> > rule_string; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_argument> > rule_argument; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_argumentList> > rule_argumentList; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_locationItem> > rule_locationItem; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_location> > rule_location; - typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_systemState> > rule_systemState; - typedef boost::spirit::classic::grammar_def<rule_systemState> type; - }; - - template <typename Scanner> - struct definition : gram_base<Scanner>::type { - typename gram_base<Scanner>::rule_hexChar _hexChar; - typename gram_base<Scanner>::rule_hexCode _hexCode; - typename gram_base<Scanner>::rule_alphaNum _alphaNum; - typename gram_base<Scanner>::rule_string _string; - typename gram_base<Scanner>::rule_argument _argument; - typename gram_base<Scanner>::rule_argumentList _argumentList; - typename gram_base<Scanner>::rule_locationItem _locationItem; - typename gram_base<Scanner>::rule_location _location; - typename gram_base<Scanner>::rule_systemState _systemState; - - definition(const SystemStateGrammar &) : - _hexChar(), - _hexCode(), - _alphaNum(), - _string(), - _argument(), - _argumentList(), - _locationItem(), - _location(), - _systemState() { - _hexChar = ( boost::spirit::classic::chset<>("A-Fa-f0-9") ); - _hexCode = ( boost::spirit::classic::ch_p('%') >> _hexChar >> _hexChar); - _alphaNum = ( boost::spirit::classic::chset<>("A-Za-z0-9") | - boost::spirit::classic::ch_p('-') | boost::spirit::classic::ch_p('.') | - boost::spirit::classic::ch_p('_') | boost::spirit::classic::ch_p('~') ); - _string = ( +( boost::spirit::classic::ch_p('+') | _hexCode | _alphaNum ) ); - _argument = ( _string >> boost::spirit::classic::ch_p('=') >> _string ); - _argumentList = ( _argument >> *( boost::spirit::classic::ch_p('&') >> _argument ) ); - _locationItem = ( boost::spirit::classic::str_p("..") | boost::spirit::classic::ch_p('.') | _string ); - _location = ( !boost::spirit::classic::ch_p('/') - >> _locationItem - >> *( boost::spirit::classic::ch_p('/') >> _locationItem ) - >> !boost::spirit::classic::ch_p('/') ); - _systemState = ( +( *boost::spirit::classic::space_p >> - _location >> !( boost::spirit::classic::ch_p('?') >> _argumentList ) ) ); - this->start_parsers(_systemState); - } - }; -}; - -template<typename T> void -debugNode(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, const string &prefix = "") -{ - std::map<boost::spirit::classic::parser_id, string> names; - names[boost::spirit::classic::parser_id(grammar.id_hexChar)] = "hexChar"; - names[boost::spirit::classic::parser_id(grammar.id_hexCode)] = "hexCode"; - names[boost::spirit::classic::parser_id(grammar.id_alphaNum)] = "alphaNum"; - names[boost::spirit::classic::parser_id(grammar.id_string)] = "string"; - names[boost::spirit::classic::parser_id(grammar.id_argument)] = "argument"; - names[boost::spirit::classic::parser_id(grammar.id_argumentList)] = "argumentList"; - names[boost::spirit::classic::parser_id(grammar.id_locationItem)] = "locationItem"; - names[boost::spirit::classic::parser_id(grammar.id_location)] = "location"; - names[boost::spirit::classic::parser_id(grammar.id_systemState)] = "systemState"; - - std::cout << prefix << names[node.value.id()] << ": " << string(node.value.begin(), node.value.end()) << std::endl; - for (size_t i = 0; i < node.children.size(); i++) { - debugNode(grammar, node.children[i], vespalib::make_string("%s %d.", prefix.c_str(), (int)i)); - } -} - -template<typename T> string -parseHexChar(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_hexChar); - assert(node.children.size() == 1); - assert(node.children[0].value.id().to_long() == grammar.id_hexChar); - (void) grammar; - return string(node.children[0].value.begin(), node.children[0].value.end()); -} - -template<typename T> char -parseHexCode(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_hexCode); - assert(node.children.size() == 3); - assert(node.children[0].value.id().to_long() == grammar.id_hexCode); - assert(node.children[1].value.id().to_long() == grammar.id_hexChar); - assert(node.children[2].value.id().to_long() == grammar.id_hexChar); - string enc = parseHexChar(grammar, node.children[1]) + parseHexChar(grammar, node.children[2]); - return (char)strtol(enc.c_str(), NULL, 16); -} - -template<typename T> string -parseAlphaNum(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_alphaNum); - assert(node.children.size() == 1); - assert(node.children[0].value.id().to_long() == grammar.id_alphaNum); - (void) grammar; - return string(node.children[0].value.begin(), node.children[0].value.end()); -} - -template<typename T> string -parseString(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_string); - string ret; - for (size_t i = 0; i < node.children.size(); ++i) { - boost::spirit::classic::tree_node<T> &child = node.children[i]; - if (child.value.id().to_long() == grammar.id_string) { - ret += " "; - } - else if (child.value.id().to_long() == grammar.id_alphaNum) { - ret += parseAlphaNum(grammar, child); - } - else if (child.value.id().to_long() == grammar.id_hexCode) { - ret += parseHexCode(grammar, child); - } - } - return ret; -} - -template<typename T> void -parseArgument(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, - std::map<string, string> &arg) -{ - assert(node.value.id().to_long() == grammar.id_argument); - assert(node.children.size() == 3); - assert(node.children[0].value.id().to_long() == grammar.id_string); - assert(node.children[1].value.id().to_long() == grammar.id_argument); - assert(node.children[2].value.id().to_long() == grammar.id_string); - string key = parseString(grammar, node.children[0]); - string val = parseString(grammar, node.children[2]); - arg[key] = val; -} - -template<typename T> void -parseArgumentList(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, - std::map<string, string> &arg) -{ - assert(node.value.id().to_long() == grammar.id_argumentList); - for (size_t i = 0; i < node.children.size(); ++i) { - boost::spirit::classic::tree_node<T> &child = node.children[i]; - if (child.value.id().to_long() == grammar.id_argument) { - parseArgument(grammar, child, arg); - } - } -} - -template<typename T> string -parseLocationItem(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_locationItem); - assert(node.children.size() == 1); - - string ret; - boost::spirit::classic::tree_node<T> &child = node.children[0]; - if (child.value.id().to_long() == grammar.id_locationItem) { - ret = string(child.value.begin(), child.value.end()); - } - else if (child.value.id().to_long() == grammar.id_string) { - ret = parseString(grammar, child); - } - return ret; -} - -template<typename T> string -parseLocation(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_location); - string ret; - for (size_t i = 0; i < node.children.size(); ++i) { - boost::spirit::classic::tree_node<T> &child = node.children[i]; - if (child.value.id().to_long() == grammar.id_locationItem) { - ret += parseLocationItem(grammar, child) + "/"; - } - } - return ret.substr(0, ret.size() - 1); -} - -template<typename T> NodeState::UP -parseSystemState(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) -{ - assert(node.value.id().to_long() == grammar.id_systemState); - NodeState::UP ret(new NodeState()); - string loc, pwd; - std::map<string, string> arg; - for (size_t i = 0; i < node.children.size(); ++i) { - boost::spirit::classic::tree_node<T> &child = node.children[i]; - if (child.value.id().to_long() == grammar.id_systemState) { - if (string(child.value.begin(), child.value.end()) != "?") { - if (!arg.empty()) { - ret->addChild(!loc.empty() ? loc : pwd, NodeState(arg)); - } - else { - pwd = loc; - } - loc.clear(); - arg.clear(); - } - } - else if (child.value.id().to_long() == grammar.id_location) { - if (!pwd.empty()) { - loc = pwd + "/"; - } - loc += parseLocation(grammar, child); - } - else if (child.value.id().to_long() == grammar.id_argumentList) { - parseArgumentList(grammar, child, arg); - } - } - if (!arg.empty()) { - ret->addChild(!loc.empty() ? loc : pwd, NodeState(arg)); - } - return ret; -} - -namespace { - vespalib::Lock _G_parseLock; -} - -SystemState::UP -SystemState::newInstance(const string &state) -{ - if (state.empty()) { - return SystemState::UP(new SystemState(NodeState::UP(new NodeState()))); - } - try { - vespalib::LockGuard guard(_G_parseLock); - SystemStateGrammar grammar; - boost::spirit::classic::tree_parse_info<> info = - boost::spirit::classic::pt_parse(static_cast<const char *>(&*state.begin()), - static_cast<const char *>(&*state.end()), - grammar.use_parser<0>()); - if (!info.full) { - string unexpected(info.stop); - unsigned int position = state.size() - unexpected.size(); - if (unexpected.size() > 10) { - unexpected = unexpected.substr(0, 10); - } - LOG(error, "Unexpected token at position %u ('%s') in query '%s'.", - position, unexpected.c_str(), state.c_str()); - } - else if (info.trees.size() != 1) { - LOG(error, "Parser returned %u trees, expected 1.", - (uint32_t)info.trees.size()); - } - else { - return SystemState::UP(new SystemState(parseSystemState(grammar, info.trees[0]))); - } - } - catch(std::exception& e) { - LOG(fatal, "SystemState::parse() internal error: %s", e.what()); - } - return SystemState::UP(); -} - -SystemState::SystemState(NodeState::UP root) : - _root(std::move(root)), - _lock(std::make_unique<vespalib::Lock>()) -{} - -SystemState::~SystemState() {} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h deleted file mode 100644 index 26f1b7fd713..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/documentapi/common.h> - -namespace vespalib { class Lock; } -namespace documentapi { - -class NodeState; - -/** - * This class is a factory to create a tree of {@link NodeState} objects from a parseable node state - * string. The naming of this class is intended to capture the fact that this annotated service tree actually - * contains the state of each service in the system. - */ -class SystemState { -private: - std::unique_ptr<NodeState> _root; - std::unique_ptr<vespalib::Lock> _lock; - - friend class SystemStateHandle; - - /** - * Constructs a new system state object to encapsulate a given root node state. This method is private; the only way - * to create a new instance is through the {@link #create} method. - * - * @param root The root node state. - */ - SystemState(std::unique_ptr<NodeState> root); - -public: - ~SystemState(); - SystemState(const SystemState &) = delete; - SystemState & operator = (const SystemState &) = delete; - /** - * Convenience typedefs. - */ - typedef std::unique_ptr<SystemState> UP; - - /** - * Creates a system state expression from a system state string. - * - * @param state The string to parse as a system state. - * @return The created node state tree. - * @throws RuntimeException Thrown if the string could not be parsed. - */ - static SystemState::UP newInstance(const string &state); -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp deleted file mode 100644 index 9ccaece4511..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "systemstatehandle.h" - -using namespace documentapi; - -SystemStateHandle::SystemStateHandle(SystemState &state) : - _state(&state), - _guard(*state._lock) -{} - -SystemStateHandle::SystemStateHandle(SystemStateHandle &&rhs) : - _state(rhs._state), - _guard(std::move(rhs._guard)) -{ - rhs._state = nullptr; -} - -SystemStateHandle & -SystemStateHandle::operator=(SystemStateHandle &&rhs) -{ - if (this != &rhs) { - _state = rhs._state; - _guard = std::move(rhs._guard); - rhs._state = nullptr; - } - return *this; -} - -SystemStateHandle::~SystemStateHandle() {} - diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h deleted file mode 100644 index f0342cfb2de..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "systemstate.h" -#include <vespa/vespalib/util/sync.h> - -namespace documentapi { - -/** - * Implements a handle to grant synchronized access to the content of a system state object. - */ -class SystemStateHandle { -private: - SystemState *_state; // The associated system state for which this object is a handler. - vespalib::LockGuard _guard; // The lock guard for the system state's lock. - - SystemStateHandle &operator=(const SystemStateHandle &) = delete; - SystemStateHandle(const SystemStateHandle &) = delete; - -public: - /** - * Creates a new system state handler object that grants access to the content of the supplied system - * state object. This handle is required to make sure that all access to the system state content is - * locked. - */ - SystemStateHandle(SystemState &state); - - /** - * Implements the move constructor. - * - * @param rhs The handle to move to this. - */ - SystemStateHandle(SystemStateHandle &&rhs); - - SystemStateHandle &operator=(SystemStateHandle &&rhs); - /** - * Destructor. Releases the contained lock on the associated system state object. There is no unlock() - * mechanism provided, since this will happen automatically as soon as this handle goes out of scope. - */ - ~SystemStateHandle(); - - /** Returns whether or not this handle is valid. */ - bool isValid() const { return _state != NULL; } - - /** Returns a reference to the root node of the associated system state. */ - NodeState &getRoot() { return *_state->_root; } - - /** Returns a const reference to the root node of the associated system state. */ - const NodeState &getRoot() const { return *_state->_root; } -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp deleted file mode 100644 index 13bd0750c31..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "urlencoder.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/stllike/asciistream.h> - -using namespace documentapi; - -const string -URLEncoder::encode(const string &str) -{ - vespalib::asciistream out; - for (size_t i = 0; i < str.size(); i++) { - char c = str[i]; - if ((c >= 48 && c <= 57) || // The range '0'-'9'. - (c >= 65 && c <= 90) || // The range 'A'-'Z'. - (c >= 97 && c <= 122) || // The range 'a'-'z'. - (c == '-' || c == '.' || c == '*' || c == '_')) { - out << c; - } - else if (c == ' ') { - out << '+'; - } - else { - out << "%" << vespalib::make_string("%02X", c & 0xff); - } - } - return out.str(); -} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h deleted file mode 100644 index 5b09327c0a0..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/documentapi/common.h> - -namespace documentapi { - -/** - * <p>Utility class for HTML form encoding. This class contains static methods for converting a String to the - * application/x-www-form-urlencoded MIME format. For more information about HTML form encoding, consult the - * HTML specification.</p> - * - * <p>When encoding a String, the following rules apply:</p> - * <ul> - * <li>The alphanumeric characters "a" through "z", "A" through "Z" and "0" through "9" remain the - * same.</li> - * <li>The special characters ".", "-", "*", and "_" remain the same.</li> - * <li>The space character " " is converted into a plus sign "+".</li> - * <li>All other characters are unsafe and are first converted into one or more bytes using some encoding - * scheme. Then each byte is represented by the 3-character string "%xy", where xy is the two-digit - * hexadecimal representation of the byte. The recommended encoding scheme to use is UTF-8. However, for - * compatibility reasons, if an encoding is not specified, then the default encoding of the platform is - * used.</li> - * </ul> - * - * <p>For example using UTF-8 as the encoding scheme the string "The string �@foo-bar" would get converted to - * "The+string+%C3%BC%40foo-bar" because in UTF-8 the character � is encoded as two bytes C3 (hex) and BC - * (hex), and the character @ is encoded as one byte 40 (hex).</p> - */ -class URLEncoder { -public: - /** - * Translates a string into application/x-www-form-urlencoded format using a UTF-8 encoding. - * - * @param str The string to be translated. - * @return The translated string. - */ - static const string encode(const string &str); -}; - -} - diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java index 38fa91b4831..f1232259ced 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainer.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; import com.google.inject.AbstractModule; @@ -17,8 +17,6 @@ import com.yahoo.jdisc.service.ServerProvider; import java.net.URI; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; /** * @author Simon Thoresen @@ -26,8 +24,6 @@ import java.util.logging.Logger; */ public class ActiveContainer extends AbstractResource implements CurrentContainer { - private static final Logger log = Logger.getLogger(ActiveContainer.class.getName()); - private final ContainerTermination termination; private final Injector guiceInjector; private final Iterable<ServerProvider> serverProviders; @@ -36,7 +32,6 @@ public class ActiveContainer extends AbstractResource implements CurrentContaine private final Map<String, BindingSet<RequestHandler>> clientBindings; private final BindingSetSelector bindingSetSelector; private final TimeoutManagerImpl timeoutMgr; - private final Destructor destructor; public ActiveContainer(ContainerBuilder builder) { serverProviders = builder.serverProviders().activate(); @@ -61,16 +56,14 @@ public class ActiveContainer extends AbstractResource implements CurrentContaine }); guiceInjector = builder.guiceModules().activate(); termination = new ContainerTermination(builder.appContext()); - destructor = new Destructor(resourceReferences, timeoutMgr, termination); } @Override protected void destroy() { - boolean alreadyDestructed = destructor.destruct(); - if (alreadyDestructed) { - throw new IllegalStateException( - "Already destructed! This should not occur unless destroy have been called directly!"); - } + resourceReferences.release(); + timeoutMgr.shutdown(); + termination.run(); + super.destroy(); } /** @@ -126,40 +119,4 @@ public class ActiveContainer extends AbstractResource implements CurrentContaine return new ContainerSnapshot(this, serverBindings, clientBindings); } - // TODO Rewrite to use cleaners after Java 9 migration - @Override - protected void finalize() throws Throwable { - boolean alreadyDestructed = destructor.destruct(); - if (!alreadyDestructed) { - log.severe(toString() + " was not correctly cleaned up " + - "because of a resource leak or invalid use of reference counting."); - } - super.finalize(); - } - - // NOTE: An instance of this class must never contain a reference to the outer class (ActiveContainer). - private static class Destructor { - private final ResourcePool resourceReferences; - private final TimeoutManagerImpl timeoutMgr; - private final ContainerTermination termination; - private final AtomicBoolean done = new AtomicBoolean(); - - private Destructor(ResourcePool resourceReferences, - TimeoutManagerImpl timeoutMgr, - ContainerTermination termination) { - this.resourceReferences = resourceReferences; - this.timeoutMgr = timeoutMgr; - this.termination = termination; - } - - boolean destruct() { - boolean alreadyDestructed = this.done.getAndSet(true); - if (!alreadyDestructed) { - resourceReferences.release(); - timeoutMgr.shutdown(); - termination.run(); - } - return alreadyDestructed; - } - } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java index b7cab923454..6c09cdf92f7 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationEnvironmentModule.java @@ -8,6 +8,7 @@ import com.yahoo.jdisc.application.ContainerBuilder; import com.yahoo.jdisc.application.ContainerThread; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; import java.util.concurrent.ThreadFactory; @@ -28,6 +29,7 @@ class ApplicationEnvironmentModule extends AbstractModule { bind(CurrentContainer.class).toInstance(loader); bind(OsgiFramework.class).toInstance(loader.osgiFramework()); bind(ThreadFactory.class).to(ContainerThread.Factory.class); + bind(ContainerWatchdogMetrics.class).toInstance(loader.getContainerWatchdogMetrics()); } @Provides diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java index 81eb5815a01..3ba48ffd8cd 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java @@ -15,6 +15,7 @@ import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.application.OsgiHeader; import com.yahoo.jdisc.service.ContainerNotReadyException; import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleException; @@ -40,6 +41,7 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C private final AtomicReference<ActiveContainer> containerRef = new AtomicReference<>(); private final Object appLock = new Object(); private final List<Bundle> appBundles = new ArrayList<>(); + private final ContainerWatchdog watchdog = new ContainerWatchdog(); private Application application; private ApplicationInUseTracker applicationInUseTracker; @@ -68,6 +70,7 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C next.retainReference(applicationInUseTracker); } + watchdog.onContainerActivation(next); prev = containerRef.getAndSet(next); if (prev == null) { return null; @@ -193,8 +196,9 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C public void destroy() { log.finer("Destroying application loader."); try { + watchdog.close(); osgiFramework.stop(); - } catch (BundleException e) { + } catch (BundleException | InterruptedException e) { e.printStackTrace(); } } @@ -205,6 +209,10 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C } } + public ContainerWatchdogMetrics getContainerWatchdogMetrics() { + return watchdog; + } + public OsgiFramework osgiFramework() { return osgiFramework; } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java new file mode 100644 index 00000000000..30aa0028465 --- /dev/null +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerWatchdog.java @@ -0,0 +1,126 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.core; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * A watchdog that monitors all deactivated {@link ActiveContainer} instances with the purpose of detecting stale containers. + * + * @author bjorncs + */ +class ContainerWatchdog implements ContainerWatchdogMetrics, AutoCloseable { + + static final Duration GRACE_PERIOD = Duration.ofMinutes(30); + static final Duration UPDATE_PERIOD = Duration.ofMinutes(5); + + private static final Logger log = Logger.getLogger(ContainerWatchdog.class.getName()); + + private final Object monitor = new Object(); + private final List<DeactivatedContainer> deactivatedContainers = new LinkedList<>(); + private final ScheduledExecutorService scheduler; + private final Clock clock; + + private ActiveContainer currentContainer; + private Instant currentContainerActivationTime; + private int numStaleContainers; + + ContainerWatchdog() { + this(new ScheduledThreadPoolExecutor( + 1, + runnable -> { + Thread thread = new Thread(runnable, "container-watchdog"); + thread.setDaemon(true); + return thread; + }), + Clock.systemUTC()); + } + + ContainerWatchdog(ScheduledExecutorService scheduler, Clock clock) { + this.scheduler = scheduler; + this.clock = clock; + scheduler.scheduleAtFixedRate( + this::monitorDeactivatedContainers, UPDATE_PERIOD.getSeconds(), UPDATE_PERIOD.getSeconds(), TimeUnit.SECONDS); + } + + @Override + public void emitMetrics(Metric metric) { + int numStaleContainers; + synchronized (monitor) { + numStaleContainers = this.numStaleContainers; + } + metric.set(TOTAL_DEACTIVATED_CONTAINERS, numStaleContainers, null); + } + + @Override + public void close() throws InterruptedException { + scheduler.shutdownNow(); + scheduler.awaitTermination(1, TimeUnit.MINUTES); + synchronized (monitor) { + deactivatedContainers.clear(); + currentContainer = null; + currentContainerActivationTime = null; + } + } + + void onContainerActivation(ActiveContainer nextContainer) { + synchronized (monitor) { + if (currentContainer != null) { + deactivatedContainers.add( + new DeactivatedContainer(currentContainer, currentContainerActivationTime, clock.instant())); + } + currentContainer = nextContainer; + currentContainerActivationTime = clock.instant(); + } + } + + void monitorDeactivatedContainers() { + synchronized (monitor) { + int numStaleContainer = 0; + Iterator<DeactivatedContainer> iterator = deactivatedContainers.iterator(); + while (iterator.hasNext()) { + DeactivatedContainer container = iterator.next(); + int refCount = container.instance.retainCount(); + if (refCount == 0) { + iterator.remove(); + break; + } + if (isPastGracePeriod(container)) { + ++numStaleContainer; + log.warning( + String.format( + "Deactivated container still alive: instance=%s, activated=%s, deactivated=%s, ref-count=%d", + container.instance.toString(), container.timeActivated, container.timeDeactivated, refCount)); + } + } + this.numStaleContainers = numStaleContainer; + } + } + + private boolean isPastGracePeriod(DeactivatedContainer container) { + return clock.instant().isAfter(container.timeDeactivated.plus(GRACE_PERIOD)); + } + + private static class DeactivatedContainer { + final ActiveContainer instance; + final Instant timeActivated; + final Instant timeDeactivated; + + DeactivatedContainer(ActiveContainer instance, Instant timeActivated, Instant timeDeactivated) { + this.instance = instance; + this.timeActivated = timeActivated; + this.timeDeactivated = timeDeactivated; + } + } +} diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ContainerWatchdogMetrics.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ContainerWatchdogMetrics.java new file mode 100644 index 00000000000..2a286aef990 --- /dev/null +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/ContainerWatchdogMetrics.java @@ -0,0 +1,18 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.statistics; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.core.ActiveContainer; + +/** + * Tracks statistics on stale {@link ActiveContainer} instances. + * + * @author bjorncs + */ +public interface ContainerWatchdogMetrics { + + String TOTAL_DEACTIVATED_CONTAINERS = "jdisc.deactivated_containers.total"; + + void emitMetrics(Metric metric); + +} diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java new file mode 100644 index 00000000000..cebe03c5103 --- /dev/null +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/statistics/package-info.java @@ -0,0 +1,4 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +@com.yahoo.osgi.annotation.ExportPackage +package com.yahoo.jdisc.statistics; diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ContainerWatchdogTest.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ContainerWatchdogTest.java new file mode 100644 index 00000000000..94e7c3a1d22 --- /dev/null +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ContainerWatchdogTest.java @@ -0,0 +1,73 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.core; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.test.TestDriver; +import com.yahoo.test.ManualClock; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * @author bjorncs + */ +public class ContainerWatchdogTest { + + @Test + public void watchdog_counts_stale_container() { + TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(); + ManualClock clock = new ManualClock(Instant.EPOCH); + DummyMetric metric = new DummyMetric(); + ContainerWatchdog watchdog = new ContainerWatchdog(mock(ScheduledExecutorService.class), clock); + + ActiveContainer containerWithoutRetainedResources = new ActiveContainer(driver.newContainerBuilder()); + + watchdog.onContainerActivation(containerWithoutRetainedResources); + assertEquals(0, runMonitorStepAndGetStaleContainerCount(watchdog, metric)); + + clock.advance(Duration.ofHours(1)); + watchdog.onContainerActivation(null); + assertEquals(0, runMonitorStepAndGetStaleContainerCount(watchdog, metric)); + + clock.advance(ContainerWatchdog.GRACE_PERIOD); + assertEquals(0, runMonitorStepAndGetStaleContainerCount(watchdog, metric)); + + clock.advance(Duration.ofSeconds(1)); + assertEquals(1, runMonitorStepAndGetStaleContainerCount(watchdog, metric)); + + containerWithoutRetainedResources.release(); + assertEquals(0, runMonitorStepAndGetStaleContainerCount(watchdog, metric)); + } + + private static int runMonitorStepAndGetStaleContainerCount(ContainerWatchdog watchdog, DummyMetric metric) { + watchdog.monitorDeactivatedContainers(); + watchdog.emitMetrics(metric); + return metric.value; + } + + private static class DummyMetric implements Metric { + int value; + + @Override + public void set(String key, Number val, Context ctx) { + this.value = val.intValue(); + } + + @Override + public void add(String key, Number val, Context ctx) { + throw new UnsupportedOperationException(); + } + + @Override + public Context createContext(Map<String, ?> properties) { + throw new UnsupportedOperationException(); + } + } +} + diff --git a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java index 374b3fc6f8c..5c6569d9b9b 100644 --- a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java @@ -5,7 +5,6 @@ import java.util.logging.Logger; import com.yahoo.log.LogLevel; import com.yahoo.logserver.handlers.archive.ArchiverPlugin; -import com.yahoo.logserver.handlers.lasterrorsholder.LastErrorsHolderPlugin; import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; import com.yahoo.logserver.handlers.replicator.ReplicatorPlugin; @@ -23,7 +22,6 @@ public class BuiltinPluginLoader extends AbstractPluginLoader { loadFromClass(ArchiverPlugin.class); loadFromClass(ReplicatorPlugin.class); loadFromClass(LogMetricsPlugin.class); - loadFromClass(LastErrorsHolderPlugin.class); log.log(LogLevel.DEBUG, "done loading builtin plugins"); } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java deleted file mode 100644 index 927486bdd72..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolder.java +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.handlers.lasterrorsholder; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.io.Connection; -import com.yahoo.io.ConnectionFactory; -import com.yahoo.io.Listener; -import com.yahoo.log.LogLevel; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.handlers.AbstractLogHandler; - -import java.io.IOException; -import java.io.StringWriter; -import java.nio.channels.SocketChannel; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - -/** - * The LastErrorsHolder handler is used for holding the last n - * messages at level error or higher. Connecting to this handler - * will return a Json object with the last errors (default is last 100 errors) - * - * @author hmusum - */ -public class LastErrorsHolder extends AbstractLogHandler implements ConnectionFactory { - - private static final Logger log = Logger.getLogger(LastErrorsHolder.class.getName()); - private static final int maxNumErrors = 100; - private final Object lock = new Object(); - - private int port; - private Listener listener; - - private final ArrayList<LogMessage> errors = new ArrayList<>(); - private int numberOfErrors = 0; - - /** - * @param port The port to which this handler listens to. - */ - public LastErrorsHolder(int port) throws IOException { - this.port = port; - listen(port); - } - - public void listen(int port) throws IOException { - if (listener != null) { - throw new IllegalStateException("already listening to port " + this.port); - } - listener = new Listener("last-errors-holder"); - listener.listen(this, port); - listener.start(); - log.log(LogLevel.CONFIG, "port=" + port); - } - - public boolean doHandle(LogMessage msg) { - if (msg.getLevel().equals(LogLevel.ERROR) || msg.getLevel().equals(LogLevel.FATAL)) { - synchronized (lock) { - numberOfErrors++; - if (errors.size() < maxNumErrors) { - errors.add(msg); - } else if (numberOfErrors == maxNumErrors) { - log.log(LogLevel.DEBUG, String.format("Not storing errors, have reached maximum number of errors: %d, total number of errors received: %d", - maxNumErrors, numberOfErrors)); - } - } - } - return true; - } - - public void close() { - try { - listener.interrupt(); - listener.join(); - log.log(LogLevel.DEBUG, "listener stopped"); - } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "listener was interrupted", e); - } - } - - public void flush() { - } - - /** - * Factory method for creating new connections. Since we just return a result - * when client connection happens, we also write the result here - * - * @param socket The new SocketChannel - * @param listener The Listener instance we want to use - */ - public Connection newConnection(SocketChannel socket, Listener listener) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "New last-errors-holder connection: " + socket); - } - LastErrorsHolderConnection connection = new LastErrorsHolderConnection(socket); - synchronized (lock) { - Messages messages = new Messages(); - for (LogMessage error : errors) { - messages.addMessage( - new Message(error.getTime()/1000, - error.getHost(), - error.getService(), - error.getLevel().getName(), - error.getPayload())); - } - messages.setNumberOfErrors(numberOfErrors); - - try { - ObjectMapper mapper = new ObjectMapper(); - StringWriter stringWriter = new StringWriter(); - mapper.writeValue(stringWriter, messages); - connection.enqueue(StandardCharsets.UTF_8.encode(stringWriter.toString())); - } catch (IOException e) { - log.log(LogLevel.WARNING, "Could not enqueue log message", e); - } - - errors.clear(); - numberOfErrors = 0; - } - - return connection; - } - - public String toString() { - return LastErrorsHolder.class.getName(); - } - - - static class Messages { - private final List<Message> messages = new ArrayList<>(); - private long errorCount = 0; // There might be more errors than number of messages - - void addMessage(Message message) { - messages.add(message); - } - - void setNumberOfErrors(long errorCount) { - this.errorCount = errorCount; - } - - public List<Message> getMessages() { - return messages; - } - - public long getErrorCount() { - return errorCount; - } - } - - static class Message { - private final long time; - private final String hostname; - private final String service; - private final String logLevel; - private final String message; - - Message(long time, String hostname, String service, String logLevel, String message) { - this.time = time; - this.hostname = hostname; - this.service = service; - this.logLevel = logLevel; - this.message = message; - } - - public long getTime() { - return time; - } - - public String getMessage() { - return message; - } - - public String getLogLevel() { - return logLevel; - } - - public String getHostname() { - return hostname; - } - - public String getService() { - return service; - } - } - -} diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java deleted file mode 100644 index c1d86859773..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderConnection.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.handlers.lasterrorsholder; - -import com.yahoo.io.Connection; -import com.yahoo.log.LogLevel; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.filter.LogFilter; -import com.yahoo.logserver.filter.LogFilterManager; -import com.yahoo.logserver.formatter.LogFormatter; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.SelectionKey; -import java.nio.channels.SocketChannel; -import java.util.logging.Logger; - -/** - * LastErrorsHandler client connection. - * - * @author hmusum - */ -public class LastErrorsHolderConnection implements Connection, LogFilter { - private static final Logger log = Logger.getLogger(LastErrorsHolderConnection.class.getName()); - - private final SocketChannel socket; - private ByteBuffer writeBuffer; - private final ByteBuffer readBuffer = ByteBuffer.allocate(4096); - private LogFilter filter = null; - protected LogFormatter formatter = null; - private static final String filterName = "system.mute"; - - /** - * Constructs a LastErrorsHolderConnection. Note that initially the - * filter of this connection is set to MuteFilter, which mutes - * all log messages. - */ - public LastErrorsHolderConnection(SocketChannel socket) { - this.socket = socket; - this.filter = LogFilterManager.getLogFilter(filterName); - } - - /** - * Check if the message is wanted by this particular replicator - * connection. The reason we provide this method is because we - * want to be able to determine if a message is wanted by any - * client before committing resources to creating a ByteBuffer to - * serialize it into. - * - * @param msg The log message offered - */ - public boolean isLoggable(LogMessage msg) { - if (filter == null) { - return true; - } - return filter.isLoggable(msg); - } - - /** - * Return the description of the currently active filter. - */ - public String description() { - if (filter == null) { - return "No filter defined"; - } - return filter.description(); - } - - - /** - * Enqueues a ByteBuffer containing the message destined - * for the client. - * - * @param buffer the ByteBuffer into which the log message is - * serialized. - */ - public synchronized void enqueue(ByteBuffer buffer) throws IOException { - writeBuffer = buffer; - write(); - } - - public void read() throws IOException { - if (!readBuffer.hasRemaining()) { - log.warning("Log message too long. Message exceeds " - + readBuffer.capacity() - + " bytes. Connection dropped."); - close(); - return; - } - - - int ret = socket.read(readBuffer); - if (ret == -1) { - close(); - return; - } - - if (ret == 0) { - if (log.isLoggable(LogLevel.INFO)) { - log.log(LogLevel.INFO, "zero byte read occurred"); - } - } - - readBuffer.flip(); - } - - public synchronized void write() throws IOException { - if (!socket.isOpen()) { - close(); - } - do { - try { - socket.write(writeBuffer); - } catch (IOException e) { - log.log(LogLevel.WARNING, "Error writing", e); - close(); - return; - } - } while (writeBuffer.hasRemaining()); - } - - public synchronized void close() throws IOException { - socket.close(); - writeBuffer = null; - } - - public int selectOps() { - return SelectionKey.OP_READ; - } - - public SocketChannel socketChannel() { - return socket; - } - - public void connect() { - } - - void setFilter(LogFilter filter) { - this.filter = filter; - } -} - diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java deleted file mode 100644 index dda396b91db..00000000000 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderPlugin.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.handlers.lasterrorsholder; - -import com.yahoo.log.LogLevel; -import com.yahoo.logserver.Server; -import com.yahoo.plugin.Config; -import com.yahoo.plugin.Plugin; - -import java.io.IOException; -import java.util.logging.Logger; - -public class LastErrorsHolderPlugin implements Plugin { - private static final String DEFAULT_PORT = "19082"; - private static final Logger log = Logger.getLogger(LastErrorsHolderPlugin.class.getName()); - private LastErrorsHolder lastErrorsHolder; - private final Server server = Server.getInstance(); - - public String getPluginName() { - return "last-errors-holder"; - } - - /** - * Initialize the plugin - */ - public void initPlugin(Config config) { - if (lastErrorsHolder != null) { - throw new IllegalStateException("plugin already initialized: " + getPluginName()); - } - int listenPort = config.getInt("port", DEFAULT_PORT); - String threadName = config.get("thread", getPluginName()); - try { - lastErrorsHolder = new LastErrorsHolder(listenPort); - } catch (IOException e) { - log.log(LogLevel.WARNING, "init failed: " + e); - return; - } - server.registerLogHandler(lastErrorsHolder, threadName); - } - - /** - * Shut down the plugin. - */ - public void shutdownPlugin() { - - if (lastErrorsHolder == null) { - throw new IllegalStateException("plugin not initialized: " + getPluginName()); - } - server.unregisterLogHandler(lastErrorsHolder); - lastErrorsHolder = null; - } -} diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java deleted file mode 100644 index 22fa7d5cf30..00000000000 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/lasterrorsholder/LastErrorsHolderTestCase.java +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.logserver.handlers.lasterrorsholder; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogLevel; -import com.yahoo.log.LogMessage; -import com.yahoo.logserver.Server; -import com.yahoo.text.Utf8; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.io.StringWriter; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SocketChannel; -import java.time.Duration; -import java.time.Instant; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -public class LastErrorsHolderTestCase { - - private static final int serverPort = 18324; - private static final int lastErrorsHolderPort = 18326; - private Server server; - private Thread serverThread; - private LastErrorsHolder lastErrorsHolder; - - @Before - public void setUp() throws InterruptedException, IOException { - server = Server.getInstance(); - server.initialize(serverPort); - serverThread = new Thread(server); - serverThread.start(); - lastErrorsHolder = new LastErrorsHolder(lastErrorsHolderPort); - } - - @After - public void tearDown() throws InterruptedException { - if (serverThread != null) { - serverThread.interrupt(); - serverThread.join(); - } - if (lastErrorsHolder != null) lastErrorsHolder.close(); - } - - public String connectAndGetLogMessages() throws InterruptedException, IOException { - SocketChannel socket = null; - Instant start = Instant.now(); - while (Instant.now().isBefore(start.plus(Duration.ofMinutes(1)))) { - try { - InetSocketAddress address = new InetSocketAddress("localhost", lastErrorsHolderPort); - socket = SocketChannel.open(address); - break; - } catch (Exception e) { - Thread.sleep(100); - } - } - if (socket == null) { - throw new RuntimeException("Could not connect to server"); - } - - ByteBuffer buf = ByteBuffer.allocateDirect(10000); - int bytesRead = socket.read(buf); - byte[] bytes = new byte[bytesRead]; - buf.position(0); - buf.get(bytes); - socket.close(); - - return Utf8.toString(bytes); - } - - - @Test - public void testLastErrorsHolder() throws IOException, InvalidLogFormatException, InterruptedException { - LastErrorsHolder.Message logMessage1 = new LastErrorsHolder.Message(1433996283, "host1.yahoo.com", "container", LogLevel.ERROR - .getName(), "foo"); - LastErrorsHolder.Message logMessage2 = new LastErrorsHolder.Message(1433996284, "host2.yahoo.com", "container", LogLevel.ERROR - .getName(), "bar"); - LastErrorsHolder.Message logMessage3 = new LastErrorsHolder.Message(1433996285, "host2.yahoo.com", "container", LogLevel.INFO - .getName(), "bar"); - - LastErrorsHolder.Messages messages = new LastErrorsHolder.Messages(); - - // No log messages yet - String logs = connectAndGetLogMessages(); - final ObjectMapper mapper = new ObjectMapper(); - StringWriter stringWriter = new StringWriter(); - mapper.writeValue(stringWriter, messages); - assertThat(logs, is(stringWriter.toString())); - - // Three messages, one is at level INFO - lastErrorsHolder.doHandle(createLogMessage(logMessage1)); - lastErrorsHolder.doHandle(createLogMessage(logMessage2)); - lastErrorsHolder.doHandle(createLogMessage(logMessage3)); - messages = new LastErrorsHolder.Messages(); - messages.addMessage(logMessage1); - messages.addMessage(logMessage2); - messages.setNumberOfErrors(2); - // Not adding logMessage3, since it is at level INFO - - logs = connectAndGetLogMessages(); - stringWriter = new StringWriter(); - mapper.writeValue(stringWriter, messages); - assertThat(logs, is(stringWriter.toString())); - } - - private LogMessage createLogMessage(LastErrorsHolder.Message message) throws InvalidLogFormatException { - return createLogMessage(message.getTime(), message.getHostname(), message.getService(), message.getLogLevel(), message - .getMessage()); - } - - private LogMessage createLogMessage(long time, String hostname, String service, String logLevel, String message) throws InvalidLogFormatException { - return LogMessage.parseNativeFormat(String.format("%d\t%s\t1/1\t%s\tcomponent\t%s\t%s", time, hostname, service, logLevel - .toLowerCase(), message)); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 479a46a38fe..218b2947a21 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -9,9 +9,13 @@ import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; import java.net.URI; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; +import java.util.function.Function; + +import static java.util.stream.Collectors.toMap; /** * Information necessary to e.g. establish communication with the config servers @@ -20,7 +24,7 @@ import java.util.stream.Collectors; */ public class ConfigServerInfo { private final List<String> configServerHostNames; - private final List<URI> configServerURIs; + private final Map<String, URI> configServerURIs; private final Optional<KeyStoreOptions> keyStoreOptions; private final Optional<KeyStoreOptions> trustStoreOptions; private final Optional<AthenzIdentity> athenzIdentity; @@ -49,7 +53,16 @@ public class ConfigServerInfo { } public List<URI> getConfigServerUris() { - return configServerURIs; + return new ArrayList<>(configServerURIs.values()); + } + + public URI getConfigServerUri(String hostname) { + URI uri = configServerURIs.get(hostname); + if (uri == null) { + throw new IllegalArgumentException("There is no config server '" + hostname + "'"); + } + + return uri; } public Optional<KeyStoreOptions> getKeyStoreOptions() { @@ -64,10 +77,13 @@ public class ConfigServerInfo { return athenzIdentity; } - private static List<URI> createConfigServerUris(String scheme, List<String> configServerHosts, int port) { - return configServerHosts.stream() - .map(hostname -> URI.create(scheme + "://" + hostname + ":" + port)) - .collect(Collectors.toList()); + private static Map<String, URI> createConfigServerUris( + String scheme, + List<String> configServerHosts, + int port) { + return configServerHosts.stream().collect(toMap( + Function.identity(), + hostname -> URI.create(scheme + "://" + hostname + ":" + port))); } private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java index 4d4a6c0328d..9f6684ba84a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java @@ -1,10 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; + import java.util.Optional; /** - * Interface to execute basic HTTP request against config server(s) + * Interface to execute basic HTTP/HTTPS request against config server(s) * * @author freva */ @@ -20,9 +22,10 @@ public interface ConfigServerApi extends AutoCloseable { <T> T delete(String path, Class<T> wantedReturnType); - /** - * Close the underlying HTTP client and any threads this class might have started. - */ + /** Set or update the socket factory */ + void setSSLConnectionSocketFactory(SSLConnectionSocketFactory sslSocketFactory); + + /** Close the underlying HTTP client and any threads this class might have started. */ @Override void close(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index c5592e91973..fd34fede291 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.node.admin.configserver; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; @@ -38,6 +40,10 @@ public class ConfigServerApiImpl implements ConfigServerApi { private final List<URI> configServerHosts; + // If this instance is associated with asynchronous updating, this field is set + // to unregister from the updater at close() + private Optional<SslConnectionSocketFactoryUpdater> socketFactoryUpdater = Optional.empty(); + /** * The 'client' will be periodically re-created by clientRefresherScheduler if we provide keyStoreOptions * or trustStoreOptions. This is needed because the key/trust stores are updated outside of node-admin, @@ -48,15 +54,66 @@ public class ConfigServerApiImpl implements ConfigServerApi { */ private volatile SelfCloseableHttpClient client; - public ConfigServerApiImpl(Collection<URI> configServerUris) { - this(configServerUris, SSLConnectionSocketFactory.getSocketFactory()); + /** + * Creates an api for talking to the config servers. + * + * <p>Registers with a SslConnectionSocketFactoryUpdater to keep the socket factory + * up to date as e.g. any client certificate expires (and unregisters at {@link #close()}) + */ + public static ConfigServerApiImpl create(ConfigServerInfo configServerInfo, + SslConnectionSocketFactoryUpdater updater) { + return createFor(updater, configServerInfo.getConfigServerUris()); + } + + /** + * Creates an api for talking to a single config server. + * + * <p>Registers with a SslConnectionSocketFactoryUpdater to keep the socket factory + * up to date as e.g. any client certificate expires (and unregisters at {@link #close()}) + */ + public static ConfigServerApiImpl createFor(ConfigServerInfo configServerInfo, + SslConnectionSocketFactoryUpdater updater, + HostName configServer) { + URI uri = configServerInfo.getConfigServerUri(configServer.value()); + return createFor(updater, Collections.singletonList(uri)); + } + + /** + * Creates an api for talking to the config servers with a fixed socket factory. + * + * <p>This may be used to avoid requiring background certificate signing requests (CSR) + * against the config server when client validation is enabled in the config server. + */ + public static ConfigServerApiImpl createWithSocketFactory( + List<URI> configServerHosts, + SSLConnectionSocketFactory socketFactory) { + return new ConfigServerApiImpl(configServerHosts, socketFactory); + } + + static ConfigServerApiImpl createForTestingWithClient(List<URI> configServerHosts, + SelfCloseableHttpClient client) { + return new ConfigServerApiImpl(configServerHosts, client); + } + + private static ConfigServerApiImpl createFor(SslConnectionSocketFactoryUpdater updater, + List<URI> configServers) { + ConfigServerApiImpl api = new ConfigServerApiImpl( + configServers, + // Passing null here (only) works because startSocketFactoryUpdating will + // set the client. This avoids an unnecessary allocation of a client. + (SelfCloseableHttpClient) null); + api.startSocketFactoryUpdating(updater); + assert api.client != null; + return api; } - ConfigServerApiImpl(Collection<URI> configServerUris, SSLConnectionSocketFactory sslConnectionSocketFactory) { - this(randomizeConfigServerUris(configServerUris), new SelfCloseableHttpClient(sslConnectionSocketFactory)); + private ConfigServerApiImpl(Collection<URI> configServerUris, + SSLConnectionSocketFactory sslConnectionSocketFactory) { + this(randomizeConfigServerUris(configServerUris), + new SelfCloseableHttpClient(sslConnectionSocketFactory)); } - ConfigServerApiImpl(List<URI> configServerHosts, SelfCloseableHttpClient client) { + private ConfigServerApiImpl(List<URI> configServerHosts, SelfCloseableHttpClient client) { this.configServerHosts = configServerHosts; this.client = client; } @@ -110,6 +167,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { + configServerHosts + ") failed, last as follows:", lastException); } + @Override public <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType) { return tryAllConfigServers(configServer -> { HttpPut put = new HttpPut(configServer.resolve(path)); @@ -121,6 +179,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { }, wantedReturnType); } + @Override public <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { return tryAllConfigServers(configServer -> { HttpPatch patch = new HttpPatch(configServer.resolve(path)); @@ -130,16 +189,19 @@ public class ConfigServerApiImpl implements ConfigServerApi { }, wantedReturnType); } + @Override public <T> T delete(String path, Class<T> wantedReturnType) { return tryAllConfigServers(configServer -> new HttpDelete(configServer.resolve(path)), wantedReturnType); } + @Override public <T> T get(String path, Class<T> wantedReturnType) { return tryAllConfigServers(configServer -> new HttpGet(configServer.resolve(path)), wantedReturnType); } + @Override public <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { return tryAllConfigServers(configServer -> { HttpPost post = new HttpPost(configServer.resolve(path)); @@ -153,6 +215,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { request.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); } + @Override public void setSSLConnectionSocketFactory(SSLConnectionSocketFactory sslSocketFactory) { this.client = new SelfCloseableHttpClient(sslSocketFactory); } @@ -164,8 +227,19 @@ public class ConfigServerApiImpl implements ConfigServerApi { return shuffledConfigServerHosts; } + private void startSocketFactoryUpdating(SslConnectionSocketFactoryUpdater updater) { + updater.registerConfigServerApi(this); + this.socketFactoryUpdater = Optional.of(updater); + } + + private void stopSocketFactoryUpdating() { + this.socketFactoryUpdater.ifPresent(updater -> updater.unregisterConfigServerApi(this)); + this.socketFactoryUpdater = Optional.empty(); + } + @Override public void close() { + stopSocketFactoryUpdating(); client.close(); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java index f52487c306f..7c15f94852b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerClients.java @@ -1,15 +1,25 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.configserver.state.State; /** + * The available (and implemented) APIs of the config server + * * @author freva */ public interface ConfigServerClients { + /** Get handle to /nodes/v2/ REST API */ NodeRepository nodeRepository(); + + /** Get handle to /orchestrator/v1/ REST API */ Orchestrator orchestrator(); + /** Get handle to the /state/v1 REST API of the specified config server */ + default State state(HostName hostname) { throw new java.lang.UnsupportedOperationException(); } + void stop(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java index 34b081689e2..1405a518625 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java @@ -1,45 +1,52 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.RealNodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.hosted.node.admin.configserver.state.State; +import com.yahoo.vespa.hosted.node.admin.configserver.state.StateImpl; -import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; /** * @author freva */ public class RealConfigServerClients implements ConfigServerClients { - private final Optional<ConfigServerApi> configServerApi; + private final SslConnectionSocketFactoryUpdater updater; + + // ConfigServerApi that talks to all config servers + private final ConfigServerApi configServerApi; + private final NodeRepository nodeRepository; private final Orchestrator orchestrator; + private final ConcurrentHashMap<HostName, State> states = new ConcurrentHashMap<>(); + private final ConfigServerInfo configServerInfo; public RealConfigServerClients(Environment environment) { - this(new SslConfigServerApiImpl(environment)); + this(environment.getConfigServerInfo(), environment.getParentHostHostname()); } + /** + * Create config server clients against a real (remote) config server. + * + * If a client certificate is required, one will be requested from the config server + * and kept up to date. On failure, this constructor will throw an exception and + * the caller may retry later. + */ public RealConfigServerClients(ConfigServerInfo info, String hostname) { - this(new SslConfigServerApiImpl(info, hostname)); - } + this.configServerInfo = info; + updater = SslConnectionSocketFactoryUpdater.createAndRefreshKeyStoreIfNeeded(info, hostname); - public RealConfigServerClients(NodeRepository nodeRepository, Orchestrator orchestrator) { - this(nodeRepository, orchestrator, Optional.empty()); - } - - private RealConfigServerClients(ConfigServerApi configServerApi) { - this(new RealNodeRepository(configServerApi), new OrchestratorImpl(configServerApi), Optional.of(configServerApi)); - } + configServerApi = ConfigServerApiImpl.create(info, updater); - private RealConfigServerClients(NodeRepository nodeRepository, Orchestrator orchestrator, - Optional<ConfigServerApi> configServerApi) { - this.nodeRepository = nodeRepository; - this.orchestrator = orchestrator; - this.configServerApi = configServerApi; + nodeRepository = new RealNodeRepository(configServerApi); + orchestrator = new OrchestratorImpl(configServerApi); } @Override @@ -53,7 +60,21 @@ public class RealConfigServerClients implements ConfigServerClients { } @Override + public State state(HostName hostname) { + return states.computeIfAbsent(hostname, this::createState); + } + + @Override public void stop() { - configServerApi.ifPresent(ConfigServerApi::close); + updater.unregisterConfigServerApi(configServerApi); + configServerApi.close(); + updater.close(); + } + + private State createState(HostName hostname) { + ConfigServerApi configServerApi = ConfigServerApiImpl.createFor( + configServerInfo, updater, hostname); + + return new StateImpl(configServerApi); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java deleted file mode 100644 index ee38541c11c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder; -import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; -import com.yahoo.vespa.hosted.node.admin.component.Environment; -import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresher; -import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import java.util.Collections; -import java.util.Optional; - -/** - * ConfigServerApi with proper keystore, truststore and hostname verifier to communicate with the - * config server(s). The keystore is refreshed automatically. - * - * @author freva - */ -public class SslConfigServerApiImpl implements ConfigServerApi { - - private final ConfigServerApiImpl configServerApi; - private final Optional<ConfigServerKeyStoreRefresher> keyStoreRefresher; - private final ConfigServerInfo configServerInfo; - - public SslConfigServerApiImpl(ConfigServerInfo configServerInfo, String hostname) { - this.configServerInfo = configServerInfo; - - // At this point we don't know the state of the keystore, it may not exist at all, or the keystore - // maybe exists, but the certificate in it is expired. Create the ConfigServerApi without a keystore - // (but with truststore and hostname verifier). - this.configServerApi = new ConfigServerApiImpl( - configServerInfo.getConfigServerUris(), makeSslConnectionSocketFactory(Optional.empty())); - - // If we have keystore options, we should make sure we use the keystore with the latest certificate, - // start the keystore refresher. - this.keyStoreRefresher = configServerInfo.getKeyStoreOptions().map(keyStoreOptions -> { - // Any callback from KeyStoreRefresher should result in using the latest keystore on disk - Runnable connectionFactoryRefresher = () -> configServerApi.setSSLConnectionSocketFactory( - makeSslConnectionSocketFactory(Optional.of(keyStoreOptions))); - - ConfigServerKeyStoreRefresher keyStoreRefresher = new ConfigServerKeyStoreRefresher( - keyStoreOptions, connectionFactoryRefresher, configServerApi, hostname); - - // Run the refresh once manually to make sure that we have a valid certificate, otherwise fail. - try { - keyStoreRefresher.refreshKeyStoreIfNeeded(); - connectionFactoryRefresher.run(); // Update connectionFactory with the keystore on disk - } catch (Exception e) { - throw new RuntimeException("Failed to acquire certificate to config server", e); - } - - keyStoreRefresher.start(); - return keyStoreRefresher; - }); - } - - public SslConfigServerApiImpl(Environment environment) { - this(environment.getConfigServerInfo(), environment.getParentHostHostname()); - - } - - @Override - public <T> T get(String path, Class<T> wantedReturnType) { - return configServerApi.get(path, wantedReturnType); - } - - @Override - public <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { - return configServerApi.post(path, bodyJsonPojo, wantedReturnType); - } - - @Override - public <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType) { - return configServerApi.put(path, bodyJsonPojo, wantedReturnType); - } - - @Override - public <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { - return configServerApi.patch(path, bodyJsonPojo, wantedReturnType); - } - - @Override - public <T> T delete(String path, Class<T> wantedReturnType) { - return configServerApi.delete(path, wantedReturnType); - } - - @Override - public void close() { - keyStoreRefresher.ifPresent(ConfigServerKeyStoreRefresher::stop); - configServerApi.close(); - } - - private SSLConnectionSocketFactory makeSslConnectionSocketFactory(Optional<KeyStoreOptions> keyStoreOptions) { - return new SSLConnectionSocketFactory(makeSslContext(keyStoreOptions), makeHostnameVerifier()); - } - - private SSLContext makeSslContext(Optional<KeyStoreOptions> keyStoreOptions) { - AthenzSslContextBuilder sslContextBuilder = new AthenzSslContextBuilder(); - configServerInfo.getTrustStoreOptions().map(KeyStoreOptions::loadKeyStore).ifPresent(sslContextBuilder::withTrustStore); - keyStoreOptions.ifPresent(options -> sslContextBuilder.withKeyStore(options.loadKeyStore(), options.password)); - - return sslContextBuilder.build(); - } - - private HostnameVerifier makeHostnameVerifier() { - return configServerInfo.getAthenzIdentity() - .map(identity -> (HostnameVerifier) new AthenzIdentityVerifier(Collections.singleton(identity))) - .orElseGet(SSLConnectionSocketFactory::getDefaultHostnameVerifier); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryCreator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryCreator.java new file mode 100644 index 00000000000..6f6bf34ae30 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryCreator.java @@ -0,0 +1,45 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; +import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder; +import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; +import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.util.Collections; +import java.util.Optional; + +/** + * @author hakon + */ +class SslConnectionSocketFactoryCreator { + SSLConnectionSocketFactory createSocketFactory( + ConfigServerInfo configServerInfo, + Optional<KeyStoreOptions> keyStoreOptions) { + SSLContext context = makeSslContext(configServerInfo, keyStoreOptions); + return new SSLConnectionSocketFactory(context, makeHostnameVerifier(configServerInfo)); + } + + private static SSLContext makeSslContext( + ConfigServerInfo configServerInfo, + Optional<KeyStoreOptions> keyStoreOptions) { + AthenzSslContextBuilder sslContextBuilder = new AthenzSslContextBuilder(); + configServerInfo.getTrustStoreOptions() + .map(KeyStoreOptions::loadKeyStore) + .ifPresent(sslContextBuilder::withTrustStore); + keyStoreOptions.ifPresent(options -> + sslContextBuilder.withKeyStore(options.loadKeyStore(), options.password)); + + return sslContextBuilder.build(); + } + + private static HostnameVerifier makeHostnameVerifier(ConfigServerInfo configServerInfo) { + return configServerInfo.getAthenzIdentity() + .map(identity -> (HostnameVerifier) new AthenzIdentityVerifier(Collections.singleton(identity))) + .orElseGet(SSLConnectionSocketFactory::getDefaultHostnameVerifier); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdater.java new file mode 100644 index 00000000000..57fa5526d73 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdater.java @@ -0,0 +1,117 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; +import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresher; +import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresherFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +/** + * Responsible for updating SSLConnectionSocketFactory on ConfigServerApiImpl asynchronously + * and as required by embedded certificate expiry + * + * @author hakon + */ +public class SslConnectionSocketFactoryUpdater implements AutoCloseable { + private final ConfigServerInfo configServerInfo; + private final SslConnectionSocketFactoryCreator socketFactoryCreator; + // Internal ConfigServerApi used to refresh the key store + private final ConfigServerApiImpl configServerApi; + private final Optional<ConfigServerKeyStoreRefresher> keyStoreRefresher; + + private final Object monitor = new Object(); + private SSLConnectionSocketFactory socketFactory = null; + private final Set<ConfigServerApi> configServerApis = new HashSet<>(); + + /** + * Creates an updater with valid initial {@link SSLConnectionSocketFactory} + * + * @param hostname the hostname of localhost + * @throws RuntimeException if e.g. key store options have been specified, but was unable + * create a create a key store with a valid certificate + */ + public static SslConnectionSocketFactoryUpdater createAndRefreshKeyStoreIfNeeded( + ConfigServerInfo configServerInfo, String hostname) { + return new SslConnectionSocketFactoryUpdater( + configServerInfo, + hostname, + ConfigServerKeyStoreRefresher::new, + new SslConnectionSocketFactoryCreator()); + } + + /** Non-private for testing only */ + SslConnectionSocketFactoryUpdater( + ConfigServerInfo configServerInfo, + String hostname, + ConfigServerKeyStoreRefresherFactory refresherFactory, + SslConnectionSocketFactoryCreator socketFactoryCreator) { + this.configServerInfo = configServerInfo; + this.socketFactoryCreator = socketFactoryCreator; + + // ConfigServerApi used to refresh the key store. Does not itself rely on a socket + // factory with key store, of course. + SSLConnectionSocketFactory socketFactoryWithoutKeyStore = + socketFactoryCreator.createSocketFactory(configServerInfo, Optional.empty()); + configServerApi = ConfigServerApiImpl.createWithSocketFactory( + configServerInfo.getConfigServerUris(), socketFactoryWithoutKeyStore); + + // If we have keystore options, we should make sure we use the keystore with the latest certificate, + // start the keystore refresher. + keyStoreRefresher = configServerInfo.getKeyStoreOptions().map(keyStoreOptions -> { + ConfigServerKeyStoreRefresher keyStoreRefresher = refresherFactory.create( + keyStoreOptions, + this::updateSslConnectionSocketFactory, + configServerApi, + hostname); + + // Run the refresh once manually to make sure that we have a valid certificate, otherwise fail. + try { + keyStoreRefresher.refreshKeyStoreIfNeeded(); + updateSslConnectionSocketFactory(); + } catch (Exception e) { + throw new RuntimeException("Failed to acquire certificate to config server", e); + } + + keyStoreRefresher.start(); + return keyStoreRefresher; + }); + } + + public SSLConnectionSocketFactory getCurrentSocketFactory() { + return socketFactory; + } + + /** Register a {@link ConfigServerApi} whose SSLConnectionSocketFactory will be kept up to date */ + public void registerConfigServerApi(ConfigServerApi configServerApi) { + synchronized (monitor) { + configServerApi.setSSLConnectionSocketFactory(socketFactory); + configServerApis.add(configServerApi); + } + } + + public void unregisterConfigServerApi(ConfigServerApi configServerApi) { + synchronized (monitor) { + configServerApis.remove(configServerApi); + } + } + + @Override + public void close() { + keyStoreRefresher.ifPresent(ConfigServerKeyStoreRefresher::stop); + configServerApi.close(); + } + + private void updateSslConnectionSocketFactory() { + synchronized (monitor) { + socketFactory = socketFactoryCreator.createSocketFactory( + configServerInfo, + configServerInfo.getKeyStoreOptions()); + + configServerApis.forEach(api -> api.setSSLConnectionSocketFactory(socketFactory)); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherFactory.java new file mode 100644 index 00000000000..d7685e737f5 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/certificate/ConfigServerKeyStoreRefresherFactory.java @@ -0,0 +1,17 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.certificate; + +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; + +/** + * @author hakon + */ +@FunctionalInterface +public interface ConfigServerKeyStoreRefresherFactory { + ConfigServerKeyStoreRefresher create( + KeyStoreOptions keyStoreOptions, + Runnable keyStoreUpdatedCallback, + ConfigServerApi configServerApi, + String hostname); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java new file mode 100644 index 00000000000..7ca7a1b30dd --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthCode.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.state; + +/** + * The healthiness of a remote Vespa server based on REST API + * + * @author hakon + */ +public enum HealthCode { + DOWN("down"), + INITIALIZING("initializing"), + UP("up"); + + private final String code; + + HealthCode(String code) { + this.code = code; + } + + public static HealthCode fromString(String code) { + return HealthCode.valueOf(code.toUpperCase()); + } + + public String asString() { + return code; + } + + @Override + public String toString() { + return asString(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java new file mode 100644 index 00000000000..ab9d0786f5a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/State.java @@ -0,0 +1,12 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.state; + +/** + * The /state/v1 REST API of the config server + * + * @author hakon + */ +public interface State { + /** Issue GET on /state/v1/health */ + HealthCode getHealth(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java new file mode 100644 index 00000000000..efeb3039379 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImpl.java @@ -0,0 +1,41 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.state; + +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; + +/** + * @author hakon + */ +public class StateImpl implements State { + private final ConfigServerApi configServerApi; + + public StateImpl(ConfigServerApi configServerApi) { + this.configServerApi = configServerApi; + } + + @Override + public HealthCode getHealth() { + HealthResponse response; + try { + response = configServerApi.get("/state/v1/health", HealthResponse.class); + } catch (RuntimeException e) { + if (causedByConnectionRefused(e)) { + return HealthCode.DOWN; + } + + throw e; + } + return HealthCode.fromString(response.status.code); + } + + private static boolean causedByConnectionRefused(Throwable throwable) { + for (Throwable cause = throwable; cause != null; cause = cause.getCause()) { + if (cause instanceof java.net.ConnectException) { + return true; + } + } + + return false; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java new file mode 100644 index 00000000000..26ad5413fb8 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/state/bindings/HealthResponse.java @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.state.bindings; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Response from /state/v1/health + * + * @author hakon + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class HealthResponse { + @JsonProperty("status") + public Status status = new Status(); + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Status { + @JsonProperty("code") + public String code = "down"; + + @Override + public String toString() { + return "Status{" + + "code='" + code + '\'' + + '}'; + } + } + + @Override + public String toString() { + return "HealthResponse{" + + "status=" + status + + '}'; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/package-info.java new file mode 100644 index 00000000000..cfd932daf0d --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.docker; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java new file mode 100644 index 00000000000..e679cd259a3 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.maintenance.acl; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java new file mode 100644 index 00000000000..f868d657795 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.maintenance; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java new file mode 100644 index 00000000000..e607a270bc8 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.nodeadmin; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index daf9010513f..26242961754 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -141,7 +141,12 @@ public class NodeAgentImpl implements NodeAgent { this.lastConverge = clock.instant(); this.loopThread = new Thread(() -> { - while (!terminated.get()) tick(); + try { + while (!terminated.get()) tick(); + } catch (Throwable t) { + logger.error("Unhandled throwable, taking down system.", t); + System.exit(234); + } }); this.loopThread.setName("tick-" + hostname); } @@ -416,11 +421,15 @@ public class NodeAgentImpl implements NodeAgent { isFrozenCopy = isFrozen; } + doAtTickStart(isFrozen); + boolean converged = false; + if (isFrozenCopy) { addDebugMessage("tick: isFrozen"); } else { try { converge(); + converged = true; } catch (OrchestratorException e) { logger.info(e.getMessage()); addDebugMessage(e.getMessage()); @@ -431,11 +440,10 @@ public class NodeAgentImpl implements NodeAgent { numberOfUnhandledException++; logger.error("Unhandled exception, ignoring.", e); addDebugMessage(e.getMessage()); - } catch (Throwable t) { - logger.error("Unhandled throwable, taking down system.", t); - System.exit(234); } } + + doAtTickEnd(converged); } // Public for testing @@ -492,6 +500,9 @@ public class NodeAgentImpl implements NodeAgent { } runLocalResumeScriptIfNeeded(node); + + doBeforeConverge(node); + // Because it's more important to stop a bad release from rolling out in prod, // we put the resume call last. So if we fail after updating the node repo attributes // but before resume, the app may go through the tenant pipeline but will halt in prod. @@ -526,6 +537,39 @@ public class NodeAgentImpl implements NodeAgent { } } + /** + * Execute at start of tick + * + * WARNING: MUST NOT throw an exception + * + * @param frozen whether the agent is frozen + */ + protected void doAtTickStart(boolean frozen) {} + + /** + * Execute at end of tick + * + * WARNING: MUST NOT throw an exception + * + * @param converged Whether the tick converged: converge() was called without exception + */ + protected void doAtTickEnd(boolean converged) {} + + /** + * Execute at end of a (so far) successful converge of an active node + * + * Method a subclass can override to execute code: + * - Called right before the node repo is updated with converged attributes, and + * Orchestrator resume() is called + * - The only way to avoid a successful converge and the update to the node repo + * and Orchestrator is to throw an exception + * - The method is only called in a tick if the node is active, not frozen, and + * there are no prior phases of the converge that fails + * + * @throws RuntimeException to fail the convergence + */ + protected void doBeforeConverge(NodeSpec node) {} + private void stopFilebeatSchedulerIfNeeded() { if (currentFilebeatRestarter.isPresent()) { currentFilebeatRestarter.get().cancel(true); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java new file mode 100644 index 00000000000..cb2cc8589f1 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java index 073978530a5..351856c4852 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java @@ -101,7 +101,7 @@ public class SystemCtl { } } - private abstract class SystemCtlCommand { + public abstract class SystemCtlCommand { private final String command; private final String unit; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java index 9485e5d6728..5df790f9105 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/AddYumRepo.java @@ -60,7 +60,7 @@ public class AddYumRepo { } // For testing - AddYumRepo(String repositoryId, + public AddYumRepo(String repositoryId, String name, String baseurl, boolean enabled, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java index a9d6d779f37..d88c6f4ab33 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java @@ -23,11 +23,9 @@ public class Yum { private static final Pattern UNKNOWN_PACKAGE_PATTERN = Pattern.compile( "(?dm)^No package ([^ ]+) available\\.$"); - private final TaskContext taskContext; private final Terminal terminal; - public Yum(TaskContext taskContext, Terminal terminal) { - this.taskContext = taskContext; + public Yum(Terminal terminal) { this.terminal = terminal; } @@ -50,7 +48,6 @@ public class Yum { String[] packages, Pattern noopPattern) { return new GenericYumCommand( - taskContext, terminal, yumCommand, Arrays.asList(packages), @@ -58,19 +55,16 @@ public class Yum { } public static class GenericYumCommand { - private final TaskContext taskContext; private final Terminal terminal; private final String yumCommand; private final List<String> packages; private final Pattern commandOutputNoopPattern; private Optional<String> enabledRepo = Optional.empty(); - private GenericYumCommand(TaskContext taskContext, - Terminal terminal, + private GenericYumCommand(Terminal terminal, String yumCommand, List<String> packages, Pattern commandOutputNoopPattern) { - this.taskContext = taskContext; this.terminal = terminal; this.yumCommand = yumCommand; this.packages = packages; @@ -87,7 +81,7 @@ public class Yum { return this; } - public boolean converge() { + public boolean converge(TaskContext taskContext) { CommandLine commandLine = terminal.newCommandLine(taskContext); commandLine.add("yum", yumCommand, "--assumeyes"); enabledRepo.ifPresent(repo -> commandLine.add("--enablerepo=" + repo)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java index 7c0b1f748f9..a4230e4dc8d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java @@ -72,7 +72,7 @@ public class ConfigServerApiImplTest { return response; }); - executor = new ConfigServerApiImpl(configServers, httpMock); + executor = ConfigServerApiImpl.createForTestingWithClient(configServers, httpMock); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdaterTest.java new file mode 100644 index 00000000000..490f45b094c --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdaterTest.java @@ -0,0 +1,58 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver; + +import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; +import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresher; +import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresherFactory; +import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.junit.Before; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author hakon + */ +public class SslConnectionSocketFactoryUpdaterTest { + private final ConfigServerInfo configServerInfo = mock(ConfigServerInfo.class); + private final String hostname = "host.oath.com"; + private final ConfigServerKeyStoreRefresherFactory refresherFactory = + mock(ConfigServerKeyStoreRefresherFactory.class); + private final ConfigServerKeyStoreRefresher refresher = + mock(ConfigServerKeyStoreRefresher.class); + private final SslConnectionSocketFactoryCreator socketFactoryCreator = + mock(SslConnectionSocketFactoryCreator.class); + private final SSLConnectionSocketFactory socketFactory = mock(SSLConnectionSocketFactory.class); + + @Before + public void setUp() { + KeyStoreOptions keyStoreOptions = mock(KeyStoreOptions.class); + when(configServerInfo.getKeyStoreOptions()).thenReturn(Optional.of(keyStoreOptions)); + when(refresherFactory.create(any(), any(), any(), any())).thenReturn(refresher); + when(socketFactoryCreator.createSocketFactory(any(), any())) + .thenReturn(socketFactory); + } + + @Test + public void testSettingOfSocketFactory() { + SslConnectionSocketFactoryUpdater updater = new SslConnectionSocketFactoryUpdater( + configServerInfo, + hostname, + refresherFactory, + socketFactoryCreator); + + assertTrue(socketFactory == updater.getCurrentSocketFactory()); + + ConfigServerApi api = mock(ConfigServerApi.class); + updater.registerConfigServerApi(api); + verify(api, times(1)).setSSLConnectionSocketFactory(socketFactory); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index d1237353e1c..269c79b125c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -4,13 +4,13 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; -import com.yahoo.vespa.hosted.node.admin.NodeSpec; import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.node.admin.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; - +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -63,7 +63,9 @@ public class RealNodeRepositoryTest { try { final int port = findRandomOpenPort(); container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable); - configServerApi = new ConfigServerApiImpl(Collections.singleton(URI.create("http://127.0.0.1:" + port))); + configServerApi = ConfigServerApiImpl.createWithSocketFactory( + Collections.singletonList(URI.create("http://127.0.0.1:" + port)), + SSLConnectionSocketFactory.getSocketFactory()); return; } catch (RuntimeException e) { lastException = e; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java new file mode 100644 index 00000000000..fcb6f6786a8 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/HealthResponseTest.java @@ -0,0 +1,54 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.state; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class HealthResponseTest { + @Test + public void deserializationOfNormalResponse() throws Exception { + String jsonResponse = "{\n" + + " \"metrics\": {\n" + + " \"snapshot\": {\n" + + " \"from\": 1.523614569023E9,\n" + + " \"to\": 1.523614629023E9\n" + + " },\n" + + " \"values\": [\n" + + " {\n" + + " \"name\": \"requestsPerSecond\",\n" + + " \"values\": {\n" + + " \"count\": 121,\n" + + " \"rate\": 2.0166666666666666\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\": \"latencySeconds\",\n" + + " \"values\": {\n" + + " \"average\": 5.537190082644628E-4,\n" + + " \"count\": 121,\n" + + " \"last\": 0.001,\n" + + " \"max\": 0.001,\n" + + " \"min\": 0,\n" + + " \"rate\": 2.0166666666666666\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"status\": {\"code\": \"up\"},\n" + + " \"time\": 1523614629451\n" + + "}"; + + HealthResponse response = deserialize(jsonResponse); + + assertEquals(response.status.code, "up"); + } + + private static HealthResponse deserialize(String json) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + return mapper.readValue(json, HealthResponse.class); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java new file mode 100644 index 00000000000..01aaa385d85 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java @@ -0,0 +1,37 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.state; + +import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; +import com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.HealthResponse; +import org.junit.Test; + +import java.net.ConnectException; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StateImplTest { + private final ConfigServerApi api = mock(ConfigServerApi.class); + private final StateImpl state = new StateImpl(api); + + @Test + public void testWhenUp() { + HealthResponse response = new HealthResponse(); + response.status.code = "up"; + when(api.get(any(), any())).thenReturn(response); + + HealthCode code = state.getHealth(); + assertEquals(HealthCode.UP, code); + } + + @Test + public void connectException() { + RuntimeException exception = new RuntimeException(new ConnectException("connection refused")); + when(api.get(any(), any())).thenThrow(exception); + + HealthCode code = state.getHealth(); + assertEquals(HealthCode.DOWN, code); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java index 39eabc5c512..e5bec3c912d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java @@ -14,8 +14,8 @@ import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; public class YumTest { - TaskContext taskContext = mock(TaskContext.class); - TestTerminal terminal = new TestTerminal(); + private TaskContext taskContext = mock(TaskContext.class); + private TestTerminal terminal = new TestTerminal(); @Before public void tearDown() { @@ -29,11 +29,11 @@ public class YumTest { 0, "foobar\nNothing to do\n"); - Yum yum = new Yum(taskContext, terminal); + Yum yum = new Yum(terminal); assertFalse(yum .install("package-1", "package-2") .enableRepo("repo-name") - .converge()); + .converge(taskContext)); } @Test @@ -43,9 +43,9 @@ public class YumTest { 0, "foobar\nNo packages marked for update\n"); - assertFalse(new Yum(taskContext, terminal) + assertFalse(new Yum(terminal) .upgrade("package-1", "package-2") - .converge()); + .converge(taskContext)); } @Test @@ -55,9 +55,9 @@ public class YumTest { 0, "foobar\nNo Packages marked for removal\n"); - assertFalse(new Yum(taskContext, terminal) + assertFalse(new Yum(terminal) .remove("package-1", "package-2") - .converge()); + .converge(taskContext)); } @Test @@ -67,10 +67,10 @@ public class YumTest { 0, "installing, installing"); - Yum yum = new Yum(taskContext, terminal); + Yum yum = new Yum(terminal); assertTrue(yum .install("package-1", "package-2") - .converge()); + .converge(taskContext)); } @Test @@ -80,11 +80,11 @@ public class YumTest { 0, "installing, installing"); - Yum yum = new Yum(taskContext, terminal); + Yum yum = new Yum(terminal); assertTrue(yum .install("package-1", "package-2") .enableRepo("repo-name") - .converge()); + .converge(taskContext)); } @Test(expected = ChildProcessFailureException.class) @@ -94,10 +94,10 @@ public class YumTest { 1, "error"); - Yum yum = new Yum(taskContext, terminal); + Yum yum = new Yum(terminal); yum.install("package-1", "package-2") .enableRepo("repo-name") - .converge(); + .converge(taskContext); fail(); } @@ -112,11 +112,11 @@ public class YumTest { "No package package-2 available.\n" + "Nothing to do\n"); - Yum yum = new Yum(taskContext, terminal); + Yum yum = new Yum(terminal); Yum.GenericYumCommand install = yum.install("package-1", "package-2", "package-3"); try { - install.converge(); + install.converge(taskContext); fail(); } catch (Exception e) { assertTrue(e.getCause() != null); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/Authorizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/Authorizer.java index 3d72c9eaca9..3601d827ac6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/Authorizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/Authorizer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; -import com.yahoo.net.HostName; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import org.apache.http.NameValuePair; @@ -17,8 +16,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.BiPredicate; -import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -31,16 +30,12 @@ public class Authorizer implements BiPredicate<Principal, URI> { private final SystemName system; private final NodeRepository nodeRepository; - private final Supplier<String> hostnameSupplier; + private final Set<String> whitelistedHostnames; - public Authorizer(SystemName system, NodeRepository nodeRepository) { - this(system, nodeRepository, HostName::getLocalhost); - } - - Authorizer(SystemName system, NodeRepository nodeRepository, Supplier<String> hostnameSupplier) { + public Authorizer(SystemName system, NodeRepository nodeRepository, Set<String> whitelistedHostnames) { this.system = system; this.nodeRepository = nodeRepository; - this.hostnameSupplier = hostnameSupplier; + this.whitelistedHostnames = whitelistedHostnames; } /** Returns whether principal is authorized to access given URI */ @@ -62,7 +57,7 @@ public class Authorizer implements BiPredicate<Principal, URI> { } // The host itself can access all resources - if (isLocalhost(principal)) { + if (whitelistedHostnames.contains(principal.getName())) { return true; } @@ -88,11 +83,6 @@ public class Authorizer implements BiPredicate<Principal, URI> { .orElse(false); } - /** Returns whether given principal is the hostname of this node */ - private boolean isLocalhost(Principal principal) { - return principal.getName().equals(hostnameSupplier.get()); - } - /** Returns whether principal can access all given resources */ private <T> boolean canAccessAll(List<T> resources, Principal principal, BiPredicate<T, Principal> predicate) { return !resources.isEmpty() && resources.stream().allMatch(resource -> predicate.test(resource, principal)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java index fcefe73a8b9..98623a5100c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java @@ -3,13 +3,16 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.google.inject.Inject; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.net.HostName; import com.yahoo.vespa.athenz.tls.X509CertificateUtils; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.restapi.v2.Authorizer; import com.yahoo.vespa.hosted.provision.restapi.v2.ErrorResponse; +import org.apache.commons.lang.StringUtils; import java.net.URI; import java.security.Principal; @@ -18,6 +21,8 @@ import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.BiPredicate; import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Authorization filter for all paths in config server. @@ -32,8 +37,17 @@ public class AuthorizationFilter implements SecurityRequestFilter { private final BiConsumer<ErrorResponse, ResponseHandler> rejectAction; @Inject - public AuthorizationFilter(Zone zone, NodeRepository nodeRepository) { - this(new Authorizer(zone.system(), nodeRepository), AuthorizationFilter::logAndReject); + public AuthorizationFilter(Zone zone, NodeRepository nodeRepository, NodeRepositoryConfig nodeRepositoryConfig) { + this( + new Authorizer( + zone.system(), + nodeRepository, + Stream.concat( + Stream.of(HostName.getLocalhost()), + Stream.of(nodeRepositoryConfig.hostnameWhitelist().split(",")) + ).filter(StringUtils::isNotEmpty).collect(Collectors.toSet())), + AuthorizationFilter::logAndReject + ); } AuthorizationFilter(BiPredicate<Principal, URI> authorizer, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/AuthorizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/AuthorizerTest.java index 330262e84be..1c23dd59ad2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/AuthorizerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/AuthorizerTest.java @@ -15,6 +15,7 @@ import org.junit.Test; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -35,7 +36,7 @@ public class AuthorizerTest { public void before() { NodeFlavors flavors = new MockNodeFlavors(); nodeRepository = new MockNodeRepository(new MockCurator(), flavors); - authorizer = new Authorizer(SystemName.main, nodeRepository, () -> "cfg1"); + authorizer = new Authorizer(SystemName.main, nodeRepository, new HashSet<>(Arrays.asList("cfg1", "cfghost1"))); { // Populate with nodes used in this test. Note that only nodes requiring node repository lookup are added here Set<String> ipAddresses = new HashSet<>(Arrays.asList("127.0.0.1", "::1")); Flavor flavor = flavors.getFlavorOrThrow("default"); @@ -102,7 +103,7 @@ public class AuthorizerTest { // Trusted services can access everything in their own system assertFalse(authorized("vespa.vespa.cd.hosting", "/")); // Wrong system - assertTrue(new Authorizer(SystemName.cd, nodeRepository).test(() -> "vespa.vespa.cd.hosting", uri("/"))); + assertTrue(new Authorizer(SystemName.cd, nodeRepository, Collections.emptySet()).test(() -> "vespa.vespa.cd.hosting", uri("/"))); assertTrue(authorized("vespa.vespa.hosting", "/")); assertTrue(authorized("vespa.vespa.hosting", "/nodes/v2/node/")); assertTrue(authorized("vespa.vespa.hosting", "/nodes/v2/node/node1")); @@ -144,6 +145,7 @@ public class AuthorizerTest { public void host_authorization() { assertTrue(authorized("cfg1", "/")); assertTrue(authorized("cfg1", "/application/v2")); + assertTrue(authorized("cfghost1", "/application/v2")); } private boolean authorized(String principal, String path) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java index c6203c76347..536c6e4c700 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.restapi.v2.filter.FilterTester.Request; import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; @@ -45,8 +46,10 @@ public class AuthorizationFilterTest { private static FilterTester filterTester(SystemName system) { Zone zone = new Zone(system, Environment.prod, RegionName.defaultName()); - return new FilterTester(new AuthorizationFilter(zone, new MockNodeRepository(new MockCurator(), - new MockNodeFlavors()))); + return new FilterTester(new AuthorizationFilter( + zone, + new MockNodeRepository(new MockCurator(), new MockNodeFlavors()), + new NodeRepositoryConfig(new NodeRepositoryConfig.Builder()))); } } diff --git a/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt b/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt index df2022f2e70..f22e6c8b241 100644 --- a/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt +++ b/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt @@ -4,5 +4,3 @@ vespa_add_executable(searchcore_vespa-dump-feed_app vespa-dump-feed.cpp DEPENDS ) -vespa_add_target_system_dependency(searchcore_vespa-dump-feed_app boost boost_system${VESPA_BOOST_LIB_SUFFIX}) -vespa_add_target_system_dependency(searchcore_vespa-dump-feed_app boost boost_filesystem${VESPA_BOOST_LIB_SUFFIX}) diff --git a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp index dd9a0a732af..a6a76bf3e7b 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp +++ b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp @@ -35,6 +35,7 @@ ThreeLevelCountWriteBuffers(EC &sse, EC &spe, EC &pe) assert(_pe.getWriteOffset() == 0); } +ThreeLevelCountWriteBuffers::~ThreeLevelCountWriteBuffers() = default; void ThreeLevelCountWriteBuffers::flush() @@ -110,6 +111,6 @@ ThreeLevelCountReadBuffers::ThreeLevelCountReadBuffers(DC &ssd, DC &spd, DC &pd) pd.setReadContext(&_rcpd); } -ThreeLevelCountReadBuffers::~ThreeLevelCountReadBuffers() {} +ThreeLevelCountReadBuffers::~ThreeLevelCountReadBuffers() = default; } diff --git a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h index 40cf598ed3a..496b8e896cb 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h +++ b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.h @@ -27,15 +27,12 @@ public: uint64_t _pFileBitSize; ThreeLevelCountWriteBuffers(EC &sse, EC &spe, EC &pe); + ~ThreeLevelCountWriteBuffers(); - void - flush(); + void flush(); // unit test method. Just pads without writing proper header - void - startPad(uint32_t ssHeaderLen, - uint32_t spHeaderLen, - uint32_t pHeaderLen); + void startPad(uint32_t ssHeaderLen, uint32_t spHeaderLen, uint32_t pHeaderLen); }; diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt index 5aa4e18cb84..13c9863aa8e 100644 --- a/storage/src/tests/bucketdb/CMakeLists.txt +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(storage_testbucketdb TEST SOURCES bucketinfotest.cpp bucketmanagertest.cpp - distribution_hash_normalizer_test.cpp initializertest.cpp judyarraytest.cpp judymultimaptest.cpp diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index 0c7122547db..d42330086f0 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -82,7 +82,6 @@ public: CPPUNIT_TEST(testConflictSetOnlyClearedAfterAllBucketRequestsDone); CPPUNIT_TEST(testRejectRequestWithMismatchingDistributionHash); CPPUNIT_TEST(testDbNotIteratedWhenAllRequestsRejected); - CPPUNIT_TEST(testReceivedDistributionHashIsNormalized); // FIXME(vekterli): test is not deterministic and enjoys failing // sporadically when running under Valgrind. See bug 5932891. @@ -1311,28 +1310,4 @@ BucketManagerTest::testDbNotIteratedWhenAllRequestsRejected() auto replies = fixture.awaitAndGetReplies(1); } -/** - * Accept bucket info requests if their distribution hash is a valid permutation - * of our own config (i.e. they are set-wise identical even though the - * ordering of nodes may differ). See VESPA-1980 for context. - */ -void -BucketManagerTest::testReceivedDistributionHashIsNormalized() -{ - ConcurrentOperationFixture fixture(*this); - document::BucketId bucket(17, 0); - fixture.setUp(WithBuckets().add(bucket, api::BucketInfo(50, 100, 200))); - - // Test is configured with 10 nodes in increasing order. Jumble the order - // around. - auto infoCmd = fixture.createFullFetchCommandWithHash( - "(0;2;1;3;9;6;4;5;8;7;0)"); - _top->sendDown(infoCmd); - auto replies = fixture.awaitAndGetReplies(1); - auto& reply = dynamic_cast<api::RequestBucketInfoReply&>(*replies[0]); - // Should NOT have been rejected despite hash not matching config order - // verbatim. - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, reply.getResult().getResult()); -} - } // storage diff --git a/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp b/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp deleted file mode 100644 index da7db8c6f4c..00000000000 --- a/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/storage/bucketdb/distribution_hash_normalizer.h> -#include <string> - -namespace storage { - -using Normalizer = DistributionHashNormalizer; - -class DistributionHashNormalizerTest : public CppUnit::TestFixture { -public: - CPPUNIT_TEST_SUITE(DistributionHashNormalizerTest); - CPPUNIT_TEST(orderNonHierarchicRootGroupNodesByDistributionKey); - CPPUNIT_TEST(mayHaveSameGroupIndexAsNodeIndex); - CPPUNIT_TEST(emitOptionalCapacityForRootGroup); - CPPUNIT_TEST(emitOptionalCapacityForSubGroups); - CPPUNIT_TEST(hierarchicGroupsAreOrderedByGroupIndex); - CPPUNIT_TEST(subgroupsOrderedOnEachNestingLevel); - CPPUNIT_TEST(distributionSpecIsCopiedVerbatim); - CPPUNIT_TEST(emptyInputYieldsEmptyOutput); - CPPUNIT_TEST(parseFailureReturnsInputVerbatim); - CPPUNIT_TEST_SUITE_END(); - - void orderNonHierarchicRootGroupNodesByDistributionKey(); - void mayHaveSameGroupIndexAsNodeIndex(); - void emitOptionalCapacityForRootGroup(); - void emitOptionalCapacityForSubGroups(); - void hierarchicGroupsAreOrderedByGroupIndex(); - void subgroupsOrderedOnEachNestingLevel(); - void distributionSpecIsCopiedVerbatim(); - void emptyInputYieldsEmptyOutput(); - void parseFailureReturnsInputVerbatim(); - -private: - DistributionHashNormalizer _normalizer; -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(DistributionHashNormalizerTest); - -void -DistributionHashNormalizerTest::orderNonHierarchicRootGroupNodesByDistributionKey() -{ - // Group index is first in list. - CPPUNIT_ASSERT_EQUAL(vespalib::string("(1;0;2;3;4;7)"), - _normalizer.normalize("(1;4;7;2;0;3)")); -} - -void -DistributionHashNormalizerTest::mayHaveSameGroupIndexAsNodeIndex() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0;0;2;3;4;7)"), - _normalizer.normalize("(0;4;7;2;0;3)")); -} - -void -DistributionHashNormalizerTest::emitOptionalCapacityForRootGroup() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0c12.5;1;2;3;4;7)"), - _normalizer.normalize("(0c12.5;1;4;7;2;3)")); -} - -void -DistributionHashNormalizerTest::emitOptionalCapacityForSubGroups() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(1c5.5;1)(2;2)(3c7;3))"), - _normalizer.normalize("(0d1|*(2;2)(1c5.5;1)(3c7;3))")); -} - -void -DistributionHashNormalizerTest::hierarchicGroupsAreOrderedByGroupIndex() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(0;0)(1;1)(3;3))"), - _normalizer.normalize("(0d1|*(3;3)(1;1)(0;0))")); -} - -void -DistributionHashNormalizerTest::subgroupsOrderedOnEachNestingLevel() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(1d3|*(2;2)(3;3))" - "(4;1)(7d2|*(5;5)(6;6)))"), - _normalizer.normalize("(0d1|*(7d2|*(6;6)(5;5))" - "(1d3|*(2;2)(3;3))(4;1))")); -} - -void -DistributionHashNormalizerTest::distributionSpecIsCopiedVerbatim() -{ - // Definitely don't want to do any ordering of the distribution spec. - CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d3|2|1|*(0;0)(1;1)(3;3))"), - _normalizer.normalize("(0d3|2|1|*(3;3)(1;1)(0;0))")); -} - -void -DistributionHashNormalizerTest::emptyInputYieldsEmptyOutput() -{ - // Technically a parse failure (only 4.2 has this behavior), but it's - // explicitly checked for in BucketManager, so let's test it explicitly - // here as well. - CPPUNIT_ASSERT_EQUAL(vespalib::string(""), _normalizer.normalize("")); -} - -// In the (unlikely) case that the parser somehow fails to capture all possible -// valid values of the distribution hash, fall back to returning the non- -// normalized string. A log warning will also be emitted (though that's not -// testable). -void -DistributionHashNormalizerTest::parseFailureReturnsInputVerbatim() -{ - CPPUNIT_ASSERT_EQUAL(vespalib::string("onkel skrue"), - _normalizer.normalize("onkel skrue")); -} - -} // storage - diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 28fccf438ff..a9b28bd2542 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <boost/assign/std/vector.hpp> // for 'operator+=()' #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storageapi/message/persistence.h> diff --git a/storage/src/vespa/storage/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/bucketdb/CMakeLists.txt index 1b59ea2290b..8200884de17 100644 --- a/storage/src/vespa/storage/bucketdb/CMakeLists.txt +++ b/storage/src/vespa/storage/bucketdb/CMakeLists.txt @@ -6,7 +6,6 @@ vespa_add_library(storage_bucketdb OBJECT bucketinfo.cpp bucketmanager.cpp bucketmanagermetrics.cpp - distribution_hash_normalizer.cpp judyarray.cpp lockablemap.cpp mapbucketdatabase.cpp diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 733cc490bd3..551f6fc726d 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bucketmanager.h" -#include "distribution_hash_normalizer.h" #include "minimumusedbitstracker.h" #include "lockablemap.hpp" #include <iomanip> @@ -522,10 +521,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac lib::ClusterState::CSP clusterState(clusterStateBundle->getDerivedClusterState(bucketSpace)); assert(clusterState.get()); - DistributionHashNormalizer normalizer; - - const auto our_hash = normalizer.normalize( - distribution->getNodeGraph().getDistributionConfigHash()); + const auto our_hash = distribution->getNodeGraph().getDistributionConfigHash(); LOG(debug, "Processing %" PRIu64 " queued request bucket info commands. " "Using cluster state '%s' and distribution hash '%s'", @@ -537,8 +533,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac for (auto it = reqs.rbegin(); it != reqs.rend(); ++it) { // Currently small requests should not be forwarded to worker thread assert((*it)->hasSystemState()); - const auto their_hash = normalizer.normalize( - (*it)->getDistributionHash()); + const auto their_hash = (*it)->getDistributionHash(); std::ostringstream error; if ((*it)->getSystemState().getVersion() > _lastClusterStateSeen) { @@ -570,8 +565,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac // If we get here, message should be failed auto reply = std::make_shared<api::RequestBucketInfoReply>(**it); - reply->setResult(api::ReturnCode( - api::ReturnCode::REJECTED, error.str())); + reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, error.str())); LOG(debug, "Rejecting request from distributor %u: %s", (*it)->getDistributor(), error.str().c_str()); diff --git a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp deleted file mode 100644 index 1e6825555b2..00000000000 --- a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.cpp +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "distribution_hash_normalizer.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <boost/spirit/include/qi.hpp> -#include <boost/spirit/include/phoenix_core.hpp> -#include <boost/spirit/include/phoenix_object.hpp> -#include <boost/fusion/include/adapt_struct.hpp> -#include <boost/optional.hpp> -#include <boost/variant/recursive_wrapper.hpp> -#include <vector> -#include <algorithm> -#include <iterator> -#include <functional> - -#include <vespa/log/bufferedlogger.h> -LOG_SETUP(".storage.bucketdb.distribution_hash_normalizer"); - -// TODO -// This code can be removed once we have a model out which ensures consistent -// ordering of nodes in the stor-distribution config. - -namespace qi = boost::spirit::qi; -namespace ascii = boost::spirit::ascii; -namespace phoenix = boost::phoenix; - -namespace { - -struct GroupSet; - -using Children = boost::variant< - std::vector<unsigned int>, - boost::recursive_wrapper<GroupSet> ->; - -struct Group { - uint16_t index; - boost::optional<double> capacity; - Children children; - ~Group() {} -}; - -struct GroupSet { - std::string distribution_spec; - std::vector<Group> subgroups; -}; - -} // anon ns - -// Fusion adaptations must be in global scope. -BOOST_FUSION_ADAPT_STRUCT( - ::Group, - (uint16_t, index) - (boost::optional<double>, capacity) - (Children, children) -) - -BOOST_FUSION_ADAPT_STRUCT( - ::GroupSet, - (std::string, distribution_spec) - (std::vector<Group>, subgroups) -) - -namespace storage { -namespace { - -// Boost.Spirit v2 grammar for parsing the output of lib::Group::getConfigHash. -template <typename Iterator> -struct HashGrammar - : qi::grammar<Iterator, Group()> -{ - HashGrammar() - : HashGrammar::base_type(group) - { - using qi::uint_; - using qi::double_; - using ascii::char_; - /* - * This grammar makes the (reasonable) assumption that you can't have - * empty groups. - * - * Quick Spirit PEG DSL syntax primer for any two arbitrary parsers - * a and b (all subcomponents of parsers are themselves parsers): - * - * 'X' : character literal match parser - * a >> b : a must be followed by b ("a b" in EBNF) - * -a : optional ("a?" in EBNF) - * a | b : a or b must match (same as in EBNF) - * +a : match 1 or more times ("a+" in EBNF) - * *a : kleene star; 0 or more times ("a*" in EBNF) - * a - b : difference; a but not b - * - * Please see Boost.Spirit docs on how these map to parser attributes - * (optional maps to boost::optional of nested attribute, + or kleene - * star maps to an iterable range (std::vector) of nested attributes, - * a | b maps to a boost::variant of the attributes of a and b, - * a >> b maps to a boost::tuple of the attributes and so on; usually - * fairly intuitive). - */ - group = - '(' - >> uint_ - >> -('c' >> double_) - >> ( +(';' >> uint_) - | subgroups - ) - >> ')'; - - subgroups = ('d' >> distr_spec >> +group); - - distr_spec = +(char_ - '('); // Match everything until open paren. - } - - qi::rule<Iterator, Group()> group; - qi::rule<Iterator, GroupSet()> subgroups; - qi::rule<Iterator, std::string()> distr_spec; -}; - -template <typename Range, typename Predicate> -auto ordered_by(const Range& range, Predicate pred) { - std::vector<typename Range::value_type> copy( - std::begin(range), std::end(range)); - std::sort(copy.begin(), copy.end(), pred); - return copy; -} - -void emit_normalized_groups(vespalib::asciistream& out, const Group& g); - -struct InOrderGroupVisitor : boost::static_visitor<void> { - vespalib::asciistream& _out; - InOrderGroupVisitor(vespalib::asciistream& out) - : _out(out) - { - } - - void operator()(const std::vector<unsigned int>& nodes) const { - for (uint16_t node : ordered_by(nodes, std::less<void>())) { - _out << ';' << node; - } - } - - void operator()(const GroupSet& gs) const { - _out << 'd' << gs.distribution_spec; - auto index_less_than = [](auto& lhs, auto& rhs) { - return lhs.index < rhs.index; - }; - // Ordering will also copy nested subgroups, but the number of known - // Vespa installations with nested subgroups is currently somewhere - // around the high end of zero. - for (auto& g : ordered_by(gs.subgroups, index_less_than)) { - emit_normalized_groups(_out, g); - } - } -}; - -void emit_normalized_groups(vespalib::asciistream& out, const Group& g) { - out << '(' << g.index; - if (g.capacity) { - out << 'c' << *g.capacity; - } - boost::apply_visitor(InOrderGroupVisitor(out), g.children); - out << ')'; -} - -} // anon ns - -// We keep the grammar around across multiple normalized() calls because -// constructing the grammar object(s) isn't free. -struct DistributionHashNormalizer::ParserImpl { - using Iterator = vespalib::string::const_iterator; - HashGrammar<Iterator> grammar; -}; - -DistributionHashNormalizer::DistributionHashNormalizer() - : _impl(std::make_unique<ParserImpl>()) -{ -} - -// Required here because of incomplete ParserImpl in header. -DistributionHashNormalizer::~DistributionHashNormalizer() -{ -} - -vespalib::string -DistributionHashNormalizer::normalize(vespalib::stringref hash) const -{ - Group root; - - auto iter = hash.begin(); - const bool ok = qi::parse(iter, hash.end(), _impl->grammar, root); - if (!ok || iter != hash.end()) { - vespalib::string hash_str = hash; // stringref might not be zero-term'd. - LOGBT(warning, hash_str.c_str(), - "Unable to parse compact distribution config " - "representation: '%s'", - hash_str.c_str()); - return hash; // Fallback to input on parse failure. - } - - vespalib::asciistream out; - emit_normalized_groups(out, root); - - return out.str(); -} - -} // storage - diff --git a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.h b/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.h deleted file mode 100644 index a3a79542265..00000000000 --- a/storage/src/vespa/storage/bucketdb/distribution_hash_normalizer.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/string.h> - -namespace storage { - -/** - * Utility class for "normalizing" a received distribution hash string into - * a representation that is ordering invariant across group and node indices. - * - * All group indices and node indices will be returned in increasing order. - * - * In the case of a parser error the original string will be returned verbatim. - */ -class DistributionHashNormalizer { - // PIMPL the parser to avoid Spirit deps in header file. - struct ParserImpl; - std::unique_ptr<ParserImpl> _impl; -public: - DistributionHashNormalizer(); - ~DistributionHashNormalizer(); - - vespalib::string normalize(vespalib::stringref hash) const; -}; - -} // storage - diff --git a/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h b/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h index 888b6d248bd..41878f09014 100644 --- a/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h +++ b/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "prioritizedbucket.h" #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <vespa/storage/distributor/maintenance/prioritizedbucket.h> #include <boost/iterator/iterator_facade.hpp> -namespace storage { -namespace distributor { +namespace storage::distributor { class BucketPriorityDatabase { @@ -69,7 +68,4 @@ public: }; } -} - - diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index 2fd8ea2701f..b9a98c1295a 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -10,7 +10,6 @@ vespa_install_script(src/start-cbinaries.sh vespa-transactionlog-inspect bin) vespa_install_script(src/start-cbinaries.sh vespa-vds-disktool bin) vespa_install_script(src/start-cbinaries.sh vespa-distributord sbin) vespa_install_script(src/start-cbinaries.sh vespa-dispatch sbin) -vespa_install_script(src/start-cbinaries.sh vespa-filedistributor sbin) vespa_install_script(src/start-cbinaries.sh vespa-proton sbin) vespa_install_script(src/start-cbinaries.sh vespa-storaged sbin) diff --git a/vespabase/README b/vespabase/README index c74c1ce385f..da27a4144ff 100644 --- a/vespabase/README +++ b/vespabase/README @@ -1,2 +1 @@ -This CVS module will hold some vespa-base related stuff, -such as monitoring and startup scripts. +This module holds some vespa-base related stuff, such as startup scripts. diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 8a4e5d5b336..ed61d730629 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -17,7 +17,6 @@ vespa_define_module( APPS src/vespa/vespaclient/spoolmaster src/vespa/vespaclient/vdsstates - src/vespa/vespaclient/vespadoclocator src/vespa/vespaclient/vesparoute ) diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore b/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore deleted file mode 100644 index 52e1033ec5b..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -vespa-doclocator -vespa-doclocator-bin diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt deleted file mode 100644 index 69bffc28092..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespaclient_vespadoclocator_app - SOURCES - application.cpp - locator.cpp - main.cpp - OUTPUT_NAME vespa-doclocator-bin - INSTALL bin - DEPENDS -) -vespa_add_target_system_dependency(vespaclient_vespadoclocator_app boost boost_program_options${VESPA_BOOST_LIB_SUFFIX}) diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp deleted file mode 100644 index 1d7d84e2b01..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "application.h" -#include <boost/program_options.hpp> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/config/common/exceptions.h> -#include <vespa/document/base/idstringexception.h> -#include <iostream> - -#include <vespa/log/log.h> -LOG_SETUP("vespadoclocator"); - - -bool -Application::printDocumentLocation(Locator &locator, const std::string &docIdStr) -{ - try { - document::DocumentId docId(docIdStr); - std::cout << "DocumentId(" << docIdStr << ") " - << "BucketId(" << locator.getBucketId(docId).getId() << ") " - << "SearchColumn(" << locator.getSearchColumn(docId) << ")" - << std::endl; - } catch (document::IdParseException &e) { - std::cerr << e.getMessage() << std::endl; - return false; - } catch (vespalib::IllegalArgumentException &e) { - std::cerr << e.getMessage() << std::endl; - return false; - } - return true; -} - -int -Application::Main() -{ - // Configure locator object. - using namespace boost::program_options; - - uint32_t numColumns = 0; - std::string configId; - std::string clusterName; - std::vector<std::string> docIds; - - options_description desc("This is a tool for resolving the target column number of a document." - "\n\n" - "The options are"); - desc.add_options() - ( "config-id,i", - value<std::string>(&configId)->default_value("client"), - "The identifier to use when subscribing to configuration." ) - - ( "cluster-name,c", - value<std::string>(&clusterName), - "The name of the search cluster in which to resolve document location." ) - - ( "document-id,d", - value< std::vector<std::string> >(&docIds), - "The identifiers of the documents to locate. " - "These can also be passed as arguments without the option prefix. " - "If none is given, this tool parses identifiers from standard in." ) - - ( "help,h", - "Shows this help page." ) - - ( "num-columns,n", - value<uint32_t>(&numColumns), - "The number of columns in the search cluster. By providing this, no configuration " - "is required, meaning you can run this tool outside of a vespa cluster." ); - - positional_options_description pos; - pos.add("document-id", -1); - - variables_map vm; - try { - store(command_line_parser(_argc, _argv).options(desc).positional(pos).run(), vm); - notify(vm); - } catch (unknown_option &e) { - std::cout << e.what() << std::endl; - return EXIT_FAILURE; - } - - if (vm.count("help") != 0) { - std::cout << desc << std::endl; - return EXIT_SUCCESS; - } - - Locator locator(numColumns); - if (vm.count("num-columns") == 0) { - try { - locator.configure(configId, clusterName); - } catch (config::InvalidConfigException &e) { - std::cerr << e.getMessage() << std::endl; - return EXIT_FAILURE; - } - } - - // Locate the documents provided. - if (docIds.empty()) { - char buf[4096]; - while (!std::cin.getline(buf, 4096).eof()) { - std::string in(buf); - if (!printDocumentLocation(locator, in)) { - return EXIT_FAILURE; - } - } - } else { - for (std::vector<std::string>::iterator it = docIds.begin(); - it != docIds.end(); ++it) - { - if (!printDocumentLocation(locator, *it)) { - return EXIT_FAILURE; - } - } - } - return EXIT_SUCCESS; -} diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h deleted file mode 100644 index 58104ec407c..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "locator.h" -#include <vespa/fastos/app.h> - -class Application : public FastOS_Application { -private: - /** - * Locates and outputs the whereabouts of the given document id. If there is a problem parsing the given - * document identifier, this method returns false. - * - * @param locator The locator to use. - * @param docId The document to locate. - * @return True if the document was located. - */ - bool printDocumentLocation(Locator &locator, const std::string &docId); - -public: - int Main() override; -}; - diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp deleted file mode 100644 index 638c83ca456..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "locator.h" -#include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/messagebus/configagent.h> -#include <vespa/messagebus/iconfighandler.h> -#include <vespa/messagebus/routing/routingspec.h> -#include <vespa/vdslib/bucketdistribution.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/config/helper/configgetter.hpp> -#include <vespa/config/common/exceptions.h> -#include <vespa/config/subscription/configuri.h> -#include <boost/tokenizer.hpp> - -typedef std::map<std::string, uint32_t> ClusterMap; -using namespace config; - -namespace { - - void - processHop(const mbus::HopSpec &hop, ClusterMap &clusters) - { - typedef boost::char_separator<char> CharSeparator; - typedef boost::tokenizer<CharSeparator> Tokenizer; - - int colIdx = -1; - for (uint32_t r = 0, len = hop.getNumRecipients(); r < len; ++r) { - Tokenizer tokens(hop.getRecipient(r), CharSeparator("/")); - Tokenizer::iterator token = tokens.begin(); - for (uint32_t t = 0; t < 2 && token != tokens.end(); ++t, ++token) { - // empty - } - if (token != tokens.end()) { - colIdx = std::max(colIdx, atoi(&token->c_str()[1])); - } - } - if (colIdx < 0) { - throw config::InvalidConfigException(vespalib::make_string("Failed to process cluster '%s'.", - hop.getName().c_str())); - } - clusters.insert(ClusterMap::value_type(hop.getName().substr(15), colIdx + 1)); - } - - void - processTable(const mbus::RoutingTableSpec &table, ClusterMap &clusters) - { - clusters.clear(); - for (uint32_t i = 0, len = table.getNumHops(); i < len; ++i) { - const mbus::HopSpec &hop = table.getHop(i); - if (hop.getName().find("search/cluster.") == 0) { - processHop(hop, clusters); - } - } - if (clusters.empty()) { - throw config::InvalidConfigException("No search clusters found to resolve document location for."); - } - } - - void - processRouting(const mbus::RoutingSpec &routing, ClusterMap &clusters) - { - const mbus::RoutingTableSpec *table = NULL; - for (uint32_t i = 0, len = routing.getNumTables(); i < len; ++i) { - const mbus::RoutingTableSpec &ref = routing.getTable(i); - if (ref.getProtocol() == documentapi::DocumentProtocol::NAME) { - table = &ref; - break; - } - } - if (table == NULL) { - throw config::InvalidConfigException("No routing table available to derive config from."); - } - processTable(*table, clusters); - } - - uint32_t - getNumColumns(const mbus::RoutingSpec &routing, const std::string &clusterName) - { - ClusterMap clusters; - processRouting(routing, clusters); - - if (clusterName.empty() && clusters.size() == 1) { - return clusters.begin()->second; - } - - ClusterMap::iterator it = clusters.find(clusterName); - if (it == clusters.end()) { - std::string str = "Cluster name must be one of "; - int i = 0, len = clusters.size(); - for (it = clusters.begin(); it != clusters.end(); ++it, ++i) - { - str.append("'").append(it->first).append("'"); - if (i < len - 2) { - str.append(", "); - } else if (i == len - 2) { - str.append(" or "); - } - } - str.append("."); - throw config::InvalidConfigException(str); - } - - return it->second; - } -} - -Locator::Locator(uint32_t numColumns) : - _factory(), - _numColumns(numColumns) -{ - // empty -} - -void -Locator::configure(const std::string &configId, const std::string &clusterName) -{ - config::ConfigUri configUri(configId); - // Configure by inspecting routing config. - struct MyCB : public mbus::IConfigHandler { - mbus::RoutingSpec mySpec; - MyCB() : mySpec() {} - bool setupRouting(const mbus::RoutingSpec &spec) override { - mySpec = spec; - return true; - } - } myCB; - mbus::ConfigAgent agent(myCB); - agent.configure(ConfigGetter<messagebus::MessagebusConfig>::getConfig(configUri.getConfigId(), configUri.getContext())); - _numColumns = getNumColumns(myCB.mySpec, clusterName); -} - -document::BucketId -Locator::getBucketId(document::DocumentId &docId) -{ - return _factory.getBucketId(docId); -} - -uint32_t -Locator::getSearchColumn(document::DocumentId &docId) -{ - vdslib::BucketDistribution dist(_numColumns, 16u); - return dist.getColumn(getBucketId(docId)); -} diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h deleted file mode 100644 index 02df3a9916f..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/document/base/documentid.h> -#include <vespa/document/bucket/bucketidfactory.h> - -class Locator { -private: - document::BucketIdFactory _factory; - uint32_t _numColumns; - -public: - /** - * Constructs a new locator object. - */ - Locator(uint32_t numColumns = 0); - - /** - * Configures this locator using the supplied configuration id and cluster name. This method will - * subscribe to some known config and attempt to retrieve the number of columns of the given search - * cluster from that. - * - * This method throws an exception if it could not be configured. - * - * @param configId The config identifier to subscribe to. - * @param clusterName The name of the search cluster to resolve locations in. - */ - void configure(const std::string &configId, - const std::string &clusterName); - - /** - * Returns the bucket id to which a document id belongs. - * - * @param docId The document id to resolve. - * @return The corresponding bucket id. - */ - document::BucketId getBucketId(document::DocumentId &docId); - - /** - * Returns the column in which the given document id belongs. - * - * @param docId The document id to resolve. - * @return The corresponding column. - */ - uint32_t getSearchColumn(document::DocumentId &docId); -}; - diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp deleted file mode 100644 index da20eb5c283..00000000000 --- a/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "application.h" - -int -main(int argc, char **argv) -{ - Application app; - return app.Entry(argc, argv); -} diff --git a/vespajlib/src/main/java/com/yahoo/text/XML.java b/vespajlib/src/main/java/com/yahoo/text/XML.java index 30f30b2a270..f4cd355b0e1 100644 --- a/vespajlib/src/main/java/com/yahoo/text/XML.java +++ b/vespajlib/src/main/java/com/yahoo/text/XML.java @@ -28,17 +28,18 @@ import org.xml.sax.SAXParseException; * @author Steinar Knutsen */ public class XML { + /** * The point of this weird class and the jumble of abstract methods is * linking the scan for characters that must be quoted into the quoting * table, and making it actual work to make them go out of sync again. */ private static abstract class LegalCharacters { + // To quote http://www.w3.org/TR/REC-xml/ : // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | // [#x10000-#x10FFFF] - final boolean isLegal(final int codepoint, final boolean escapeLow, - final int stripCodePoint, final boolean isAttribute) { + final boolean isLegal(int codepoint, boolean escapeLow, int stripCodePoint, boolean isAttribute) { if (codepoint == stripCodePoint) { return removeCodePoint(); } else if (codepoint < ' ') { @@ -75,7 +76,7 @@ public class XML { } } - private boolean quotCodePoint(final boolean isAttribute) { + private boolean quotCodePoint(boolean isAttribute) { if (isAttribute) { quoteQuot(); return false; @@ -84,7 +85,7 @@ public class XML { } } - private boolean filterCodePoint(final int codepoint) { + private boolean filterCodePoint(int codepoint) { replace(codepoint); return false; } @@ -104,7 +105,7 @@ public class XML { return false; } - private boolean ctrlEscapeCodePoint(final int codepoint) { + private boolean ctrlEscapeCodePoint(int codepoint) { ctrlEscape(codepoint); return false; } @@ -349,8 +350,7 @@ public class XML { * </ul> * with character entities. * - * @param stripCodePoint - * any occurrence of this character is removed from the string + * @param stripCodePoint any occurrence of this character is removed from the string */ public static String xmlEscape(String string, boolean isAttribute, boolean escapeLowAscii, StringBuilder buffer, int stripCodePoint) { @@ -399,8 +399,7 @@ public class XML { /** * Returns the Document of an XML file reader * - * @throws RuntimeException - * if the root Document cannot be returned + * @throws RuntimeException if the root Document cannot be returned */ public static Document getDocument(Reader reader) { try { |