diff options
197 files changed, 1599 insertions, 1277 deletions
diff --git a/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java b/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java index 303ce3e8d29..fa9dfe69c5d 100644 --- a/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java +++ b/application-preprocessor/src/main/java/com/yahoo/application/preprocessor/ApplicationPreprocessor.java @@ -39,7 +39,7 @@ public class ApplicationPreprocessor { FilesApplicationPackage.Builder applicationPackageBuilder = new FilesApplicationPackage.Builder(applicationDir); outputDir.ifPresent(applicationPackageBuilder::preprocessedDir); ApplicationPackage preprocessed = applicationPackageBuilder.build().preprocess( - ZoneId.from(environment.orElse(Environment.defaultEnvironment()), region.orElse(RegionName.defaultName())), + new Zone(environment.orElse(Environment.defaultEnvironment()), region.orElse(RegionName.defaultName())), (a, b) -> { }, logger); preprocessed.validateXML(); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 330692abb1e..97322fc1c55 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -15,7 +15,6 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; -import com.yahoo.config.provision.ZoneId; import com.yahoo.path.Path; import com.yahoo.io.HexDump; import com.yahoo.io.IOUtils; @@ -650,7 +649,7 @@ public class FilesApplicationPackage implements ApplicationPackage { return searchDefinitionContents(); } - private void preprocessXML(File destination, File inputXml, ZoneId zone) throws ParserConfigurationException, TransformerException, SAXException, IOException { + private void preprocessXML(File destination, File inputXml, Zone zone) throws ParserConfigurationException, TransformerException, SAXException, IOException { Document document = new XmlPreProcessor(appDir, inputXml, zone.environment(), zone.region()).run(); Transformer transformer = TransformerFactory.newInstance().newTransformer(); try (FileOutputStream outputStream = new FileOutputStream(destination)) { @@ -659,7 +658,7 @@ public class FilesApplicationPackage implements ApplicationPackage { } @Override - public ApplicationPackage preprocess(ZoneId zone, RuleConfigDeriver ignored, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { + public ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ignored, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { IOUtils.recursiveDeleteDir(preprocessedDir); IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(".preprocessed") && ! name.equals(SERVICES) && diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 2dfaf440084..89a5769ff0c 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -2,12 +2,10 @@ package com.yahoo.config.model.application.provider; import com.yahoo.config.application.TestBase; -import com.yahoo.config.application.api.RuleConfigDeriver; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.config.provision.ZoneId; import com.yahoo.io.IOUtils; import org.junit.Rule; import org.junit.Test; @@ -42,7 +40,7 @@ public class FilesApplicationPackageTest { assertTrue(new File(appDir, "hosts.xml").exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); - ApplicationPackage processed = app.preprocess(ZoneId.from(Environment.dev, RegionName.defaultName()), + ApplicationPackage processed = app.preprocess(new Zone(Environment.dev, RegionName.defaultName()), (ruleBaseDir, outputDir) -> {}, new BaseDeployLogger()); assertTrue(new File(appDir, ".preprocessed").exists()); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 0e50b39a319..7506c884715 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -4,10 +4,10 @@ package com.yahoo.config.application.api; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; -import com.yahoo.config.provision.ZoneId; import com.yahoo.path.Path; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.text.XML; import com.yahoo.vespa.config.ConfigDefinitionKey; import org.w3c.dom.Element; @@ -15,8 +15,17 @@ import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; -import java.io.*; -import java.util.*; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -256,25 +265,15 @@ public interface ApplicationPackage { * application package. This is the entry point for the multi environment application package support. This method * will not mutate the existing application package. * - * @param zone A valid {@link ZoneId} instance, used to decide which parts of services to keep and remove + * @param zone A valid {@link Zone} instance, used to decide which parts of services to keep and remove * @param ruleConfigDeriver ignored * @param logger A {@link DeployLogger} to add output that will be returned to the user * * @return A new application package instance pointing to a new location */ - default ApplicationPackage preprocess(ZoneId zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) - throws IOException, TransformerException, ParserConfigurationException, SAXException { + default ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) + throws IOException, TransformerException, ParserConfigurationException, SAXException { throw new UnsupportedOperationException("This application package does not support preprocessing"); } - /** - * @deprecated pass a ZoneId as first parameter instead - */ - // TODO: Remove on Vespa 7 - @Deprecated - default ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) - throws IOException, TransformerException, ParserConfigurationException, SAXException { - return preprocess(zone.id(), ruleConfigDeriver, logger); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 4383e55e45d..28a54771c21 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -220,6 +220,7 @@ public final class ContainerCluster addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricUpdater"); + addSimpleComponent(com.yahoo.container.jdisc.LoggingRequestHandler.Context.class); addSimpleComponent(com.yahoo.metrics.simple.MetricManager.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); addSimpleComponent(com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); addSimpleComponent("com.yahoo.container.jdisc.state.StateMonitor"); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java index 26c20d56d63..fbbdb1df635 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java @@ -20,7 +20,8 @@ public class Zone { private final SystemName systemName; private final FlavorDefaults flavorDefaults; private final Optional<NodeFlavors> nodeFlavors; - private final ZoneId id; + private final Environment environment; + private final RegionName region; @Inject public Zone(ConfigserverConfig configserverConfig, NodeFlavors nodeFlavors) { @@ -46,25 +47,21 @@ public class Zone { RegionName region, FlavorDefaults flavorDefaults, NodeFlavors nodeFlavors) { - this.id = ZoneId.from(environment, region); + this.environment = environment; + this.region = region; this.flavorDefaults = flavorDefaults; this.systemName = systemName; this.nodeFlavors = Optional.ofNullable(nodeFlavors); } - /** Returns the id of this */ - public ZoneId id() { - return id; - } - /** Returns the current environment */ public Environment environment() { - return id.environment(); + return environment; } /** Returns the current region */ public RegionName region() { - return id.region(); + return region; } /** Returns the current system */ @@ -83,19 +80,21 @@ public class Zone { @Override public String toString() { - return id.toString(); + return "zone " + environment + "." + region; } @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof Zone)) return false; - return Objects.equals(id, ((Zone) o).id); + if (!(o instanceof Zone)) return false; + Zone zone = (Zone) o; + return environment == zone.environment && + Objects.equals(region, zone.region); } @Override public int hashCode() { - return id.hashCode(); + return Objects.hash(environment, region); } private static class FlavorDefaults { diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index cbc2317da2d..9072a20c006 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -27,6 +27,7 @@ payloadCompressionType enum { UNCOMPRESSED, LZ4 } default=LZ4 serverId string default="localhost" hostedVespa bool default=false numParallelTenantLoaders int default=4 +zookeeperLocalhostAffinity bool default=false # Zone information environment string default="prod" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index c19ae86157a..5005ddf309d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -126,7 +126,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { log.log(LogLevel.DEBUG, "Trying to acquire lock " + activateLock + " for session " + sessionId); boolean acquired = activateLock.acquire(timeoutBudget, ignoreLockFailure); if ( ! acquired) { - throw new InternalServerException("Did not get activate lock for session " + sessionId + " within " + timeout); + throw new ActivationConflictException("Did not get activate lock for session " + sessionId + " within " + timeout); } log.log(LogLevel.DEBUG, "Lock acquired " + activateLock + " for session " + sessionId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java index 3ec4d1b6e46..94707635950 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java @@ -23,14 +23,14 @@ import java.util.concurrent.Executor; public class HttpGetConfigHandler extends HttpHandler { private final RequestHandler requestHandler; - public HttpGetConfigHandler(Executor executor, RequestHandler requestHandler, AccessLog accessLog) { - super(executor, accessLog); + public HttpGetConfigHandler(HttpHandler.Context ctx, RequestHandler requestHandler) { + super(ctx); this.requestHandler = requestHandler; } @Inject - public HttpGetConfigHandler(Executor executor, Tenants tenants, AccessLog accesslog) { - this(executor, tenants.defaultTenant().getRequestHandler(), accesslog); + public HttpGetConfigHandler(HttpHandler.Context ctx, Tenants tenants) { + this(ctx, tenants.defaultTenant().getRequestHandler()); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index cc78c2715e2..e8db448b245 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; +import com.google.inject.Inject; + import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -25,8 +27,8 @@ import java.util.concurrent.Executor; */ public class HttpHandler extends LoggingRequestHandler { - public HttpHandler(Executor executor, AccessLog accessLog) { - super(executor, accessLog); + public HttpHandler(HttpHandler.Context ctx) { + super(ctx); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandler.java index 5ea0b38c110..64361c0771c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandler.java @@ -32,12 +32,12 @@ public class HttpListConfigsHandler extends HttpHandler { private final RequestHandler requestHandler; @Inject - public HttpListConfigsHandler(Executor executor, AccessLog accessLog, Tenants tenants) { - this(executor, accessLog, tenants.defaultTenant().getRequestHandler()); + public HttpListConfigsHandler(HttpHandler.Context ctx, Tenants tenants) { + this(ctx, tenants.defaultTenant().getRequestHandler()); } - public HttpListConfigsHandler(Executor executor, AccessLog accessLog, RequestHandler requestHandler) { - super(executor, accessLog); + public HttpListConfigsHandler(HttpHandler.Context ctx, RequestHandler requestHandler) { + super(ctx); this.requestHandler = requestHandler; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListNamedConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListNamedConfigsHandler.java index 7c51fd131ff..81163d79341 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListNamedConfigsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListNamedConfigsHandler.java @@ -25,14 +25,16 @@ import java.util.concurrent.Executor; public class HttpListNamedConfigsHandler extends HttpHandler { private final RequestHandler requestHandler; - public HttpListNamedConfigsHandler(Executor executor, RequestHandler requestHandler, AccessLog accessLog) { - super(executor, accessLog); + public HttpListNamedConfigsHandler(HttpHandler.Context ctx, + RequestHandler requestHandler) { + super(ctx); this.requestHandler = requestHandler; } @Inject - public HttpListNamedConfigsHandler(Executor executor, Tenants tenants, AccessLog accessLog) { - this(executor, tenants.defaultTenant().getRequestHandler(), accessLog); + public HttpListNamedConfigsHandler(HttpHandler.Context ctx, + Tenants tenants) { + this(ctx, tenants.defaultTenant().getRequestHandler()); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java index 40ffc8e9da3..5acb6e81a83 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; +import com.google.inject.Inject; + import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.logging.AccessLog; @@ -27,8 +29,9 @@ public class SessionHandler extends HttpHandler { protected final ApplicationRepository applicationRepository; - public SessionHandler(Executor executor, AccessLog accessLog, ApplicationRepository applicationRepository) { - super(executor, accessLog); + public SessionHandler(HttpHandler.Context ctx, ApplicationRepository applicationRepository) + { + super(ctx); this.applicationRepository = applicationRepository; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java new file mode 100644 index 00000000000..fb1108c18c2 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.config.server.http.status; + +import com.google.inject.Inject; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.provision.Version; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.http.JSONResponse; + +import static com.yahoo.jdisc.http.HttpResponse.Status.OK; + +/** + * Status handler that outputs config server config and config model versions in use + * + * @author hmusum + */ +public class StatusHandler extends HttpHandler { + + private final GlobalComponentRegistry componentRegistry; + + @Inject + public StatusHandler(Context ctx, GlobalComponentRegistry componentRegistry) { + super(ctx); + this.componentRegistry = componentRegistry; + } + + @Override + public HttpResponse handleGET(HttpRequest req) { + return new StatusResponse(OK, componentRegistry); + } + + private static class StatusResponse extends JSONResponse { + + StatusResponse(int status, GlobalComponentRegistry componentRegistry) { + super(status); + + Cursor configCursor = object.setObject("configserverConfig"); + SlimeUtils.copyObject(ConfigPayload.fromInstance(componentRegistry.getConfigserverConfig()).getSlime().get(), + configCursor); + + Cursor modelVersionsCursor = object.setArray("modelVersions"); + componentRegistry.getModelFactoryRegistry().getFactories().stream() + .map(ModelFactory::getVersion) + .map(Version::toString) + .forEach(modelVersionsCursor::addString); + } + + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index ef122147d79..819f1a35cf3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.google.inject.Inject; + import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -37,11 +39,11 @@ public class ApplicationHandler extends HttpHandler { private final Zone zone; private final ApplicationRepository applicationRepository; - public ApplicationHandler(Executor executor, - AccessLog accessLog, + @Inject + public ApplicationHandler(HttpHandler.Context ctx, Zone zone, ApplicationRepository applicationRepository) { - super(executor, accessLog); + super(ctx); this.zone = zone; this.applicationRepository = applicationRepository; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index 2acaa67baef..13933544ad1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -28,8 +29,10 @@ public class HostHandler extends HttpHandler { final HostRegistries hostRegistries; private final Zone zone; - public HostHandler(Executor executor, AccessLog accessLog, GlobalComponentRegistry globalComponentRegistry) { - super(executor, accessLog); + @Inject + public HostHandler(HttpHandler.Context ctx, + GlobalComponentRegistry globalComponentRegistry) { + super(ctx); this.hostRegistries = globalComponentRegistry.getHostRegistries(); this.zone = globalComponentRegistry.getZone(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java index 1b566fbb9c5..0ca720c9710 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -27,8 +27,10 @@ public class HttpGetConfigHandler extends HttpHandler { private final Tenants tenants; @Inject - public HttpGetConfigHandler(Executor executor, AccessLog accesslog, Tenants tenants) { - super(executor, accesslog); + public HttpGetConfigHandler(HttpHandler.Context ctx, + Tenants tenants) + { + super(ctx); this.tenants = tenants; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java index ea3a1a2c9f4..2a9e2b1ecf4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java @@ -34,8 +34,10 @@ public class HttpListConfigsHandler extends HttpHandler { private final Zone zone; @Inject - public HttpListConfigsHandler(Executor executor, AccessLog accesslog, Tenants tenants, Zone zone) { - super(executor, accesslog); + public HttpListConfigsHandler(HttpHandler.Context ctx, + Tenants tenants, Zone zone) + { + super(ctx); this.tenants = tenants; this.zone = zone; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java index 2262b8bc722..0a55d3585e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java @@ -29,8 +29,10 @@ public class HttpListNamedConfigsHandler extends HttpHandler { private final Zone zone; @Inject - public HttpListNamedConfigsHandler(Executor executor, AccessLog accesslog, Tenants tenants, Zone zone) { - super(executor, accesslog); + public HttpListNamedConfigsHandler(HttpHandler.Context ctx, + Tenants tenants, Zone zone) + { + super(ctx); this.tenants = tenants; this.zone = zone; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java index 79f551c270b..42872881088 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.google.inject.Inject; import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.yahoo.config.provision.TenantName; @@ -29,8 +30,11 @@ import java.util.concurrent.Executor; public class ListApplicationsHandler extends HttpHandler { private final Tenants tenants; private final Zone zone; - public ListApplicationsHandler(Executor executor, AccessLog accessLog, Tenants tenants, Zone zone) { - super(executor, accessLog); + + @Inject + public ListApplicationsHandler(HttpHandler.Context ctx, + Tenants tenants, Zone zone) { + super(ctx); this.tenants = tenants; this.zone = zone; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index f1c75ff0a01..b2330ebd97f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -33,12 +33,11 @@ public class SessionActiveHandler extends SessionHandler { private final Zone zone; @Inject - public SessionActiveHandler(Executor executor, - AccessLog accessLog, + public SessionActiveHandler(SessionHandler.Context ctx, + ApplicationRepository applicationRepository, Tenants tenants, - Zone zone, - ApplicationRepository applicationRepository) { - super(executor, accessLog, applicationRepository); + Zone zone) { + super(ctx, applicationRepository); this.tenants = tenants; this.zone = zone; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java index c9d5407e0e3..524eb01e625 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java @@ -28,11 +28,11 @@ public class SessionContentHandler extends SessionHandler { private final ContentHandler contentHandler = new ContentHandler(); @Inject - public SessionContentHandler(Executor executor, - AccessLog accessLog, - Tenants tenants, - ApplicationRepository applicationRepository) { - super(executor, accessLog, applicationRepository); + public SessionContentHandler(SessionHandler.Context ctx, + ApplicationRepository applicationRepository, + Tenants tenants) + { + super(ctx, applicationRepository); this.tenants = tenants; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 5908851e399..b0c251f477c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -49,12 +49,11 @@ public class SessionCreateHandler extends SessionHandler { private final Duration zookeeperBarrierTimeout; @Inject - public SessionCreateHandler(Executor executor, - AccessLog accessLog, + public SessionCreateHandler(SessionHandler.Context ctx, + ApplicationRepository applicationRepository, Tenants tenants, - ConfigserverConfig configserverConfig, - ApplicationRepository applicationRepository) { - super(executor, accessLog, applicationRepository); + ConfigserverConfig configserverConfig) { + super(ctx, applicationRepository); this.tenants = tenants; this.zookeeperBarrierTimeout = Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index 03a3f3556e4..2b432a50ee1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -41,12 +41,11 @@ public class SessionPrepareHandler extends SessionHandler { private final Duration zookeeperBarrierTimeout; @Inject - public SessionPrepareHandler(Executor executor, - AccessLog accessLog, + public SessionPrepareHandler(SessionHandler.Context ctx, + ApplicationRepository applicationRepository, Tenants tenants, - ConfigserverConfig configserverConfig, - ApplicationRepository applicationRepository) { - super(executor, accessLog, applicationRepository); + ConfigserverConfig configserverConfig) { + super(ctx, applicationRepository); this.tenants = tenants; this.zookeeperBarrierTimeout = Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java index 5c1d8a36f6a..955bba5f5b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.http.v2; import java.util.List; import java.util.concurrent.Executor; +import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -29,8 +30,10 @@ public class TenantHandler extends HttpHandler { private static final String TENANT_NAME_REGEXP = "[\\w-]+"; private final Tenants tenants; - public TenantHandler(Executor executor, AccessLog accessLog, Tenants tenants) { - super(executor, accessLog); + @Inject + public TenantHandler(HttpHandler.Context ctx, + Tenants tenants) { + super(ctx); this.tenants = tenants; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java deleted file mode 100644 index 1aabbb9aeb3..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.restapi.impl; - -import com.google.common.annotations.Beta; -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.ModelFactory; -import com.yahoo.config.provision.Version; -import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.http.v2.HttpGetConfigHandler; -import com.yahoo.vespa.config.server.http.v2.HttpListConfigsHandler; -import com.yahoo.vespa.config.server.http.v2.HttpListNamedConfigsHandler; -import com.yahoo.vespa.config.server.http.v2.SessionActiveHandler; -import com.yahoo.vespa.config.server.http.v2.SessionContentHandler; -import com.yahoo.vespa.config.server.http.v2.SessionCreateHandler; -import com.yahoo.vespa.config.server.http.v2.SessionPrepareHandler; -import com.yahoo.vespa.config.server.restapi.resources.StatusInformation; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import java.util.List; -import java.util.stream.Collectors; - -/** - * A simple status handler that can provide the status of the config server. - * - * @author lulf - * @since 5.1 - */ -@Beta -@Path("/") -@Produces(MediaType.APPLICATION_JSON) -public class StatusResource { - private final ConfigserverConfig configserverConfig; - private final List<String> modelVersions; - - @SuppressWarnings("UnusedParameters") - public StatusResource(@Component SessionCreateHandler create, - @Component SessionContentHandler content, - @Component SessionPrepareHandler prepare, - @Component SessionActiveHandler active, - @Component HttpGetConfigHandler getHandler, - @Component HttpListConfigsHandler listHandler, - @Component HttpListNamedConfigsHandler listNamedHandler, - @Component GlobalComponentRegistry componentRegistry) { - this.configserverConfig = componentRegistry.getConfigserverConfig(); - this.modelVersions = componentRegistry.getModelFactoryRegistry().getFactories().stream() - .map(ModelFactory::getVersion).map(Version::toString).collect(Collectors.toList()); - } - - @GET - public StatusInformation getStatus() { - return new StatusInformation(configserverConfig, modelVersions); - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/package-info.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/package-info.java deleted file mode 100644 index 5b17eed4b7f..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.config.server.restapi.impl; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/StatusInformation.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/StatusInformation.java deleted file mode 100644 index 5acd56c252e..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/StatusInformation.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.restapi.resources; - -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Status information of config server. Currently needs to convert generated configserver config to a POJO that can - * be serialized to JSON. - * - * @author lulf - * @since 5.21 - */ -public class StatusInformation { - - public ConfigserverConfig configserverConfig; - public List<String> modelVersions; - - public StatusInformation(com.yahoo.cloud.config.ConfigserverConfig configserverConfig, List<String> modelVersions) { - this.configserverConfig = new ConfigserverConfig(configserverConfig); - this.modelVersions = modelVersions; - } - - public static class ConfigserverConfig { - public final int rpcport; - public final int numthreads; - public final String zookeepercfg; - public final Collection<ZooKeeperServer> zookeeeperserver; - public final long zookeeperBarrierTimeout; - public final Collection<String> configModelPluginDir; - public final String configServerDBDir; - public final int maxgetconfigclients; - public final long sessionLifetime; - public final String applicationDirectory; - public final long masterGeneration; - public final boolean multitenant; - public final int numDelayedResponseThreads; - public final com.yahoo.cloud.config.ConfigserverConfig.PayloadCompressionType.Enum payloadCompressionType; - public final boolean useVespaVersionInRequest; - public final String serverId; - public final String region; - public final String environment; - - - public ConfigserverConfig(com.yahoo.cloud.config.ConfigserverConfig configserverConfig) { - this.rpcport = configserverConfig.rpcport(); - this.numthreads = configserverConfig.numthreads(); - this.zookeepercfg = getDefaults().underVespaHome(configserverConfig.zookeepercfg()); - this.zookeeeperserver = configserverConfig.zookeeperserver().stream() - .map(zks -> new ZooKeeperServer(zks.hostname(), zks.port())) - .collect(Collectors.toList()); - this.zookeeperBarrierTimeout = configserverConfig.zookeeper().barrierTimeout(); - this.configModelPluginDir = configserverConfig.configModelPluginDir(); - this.configServerDBDir = getDefaults().underVespaHome(configserverConfig.configServerDBDir()); - this.maxgetconfigclients = configserverConfig.maxgetconfigclients(); - this.sessionLifetime = configserverConfig.sessionLifetime(); - this.applicationDirectory = getDefaults().underVespaHome(configserverConfig.applicationDirectory()); - this.masterGeneration = configserverConfig.masterGeneration(); - this.multitenant = configserverConfig.multitenant(); - this.numDelayedResponseThreads = configserverConfig.numDelayedResponseThreads(); - this.payloadCompressionType = configserverConfig.payloadCompressionType(); - this.useVespaVersionInRequest = configserverConfig.useVespaVersionInRequest(); - this.serverId = configserverConfig.serverId(); - this.region = configserverConfig.region(); - this.environment = configserverConfig.environment(); - } - } - - public static class ZooKeeperServer { - public final String hostname; - public final int port; - - public ZooKeeperServer(String hostname, int port) { - this.hostname = hostname; - this.port = port; - } - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/package-info.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/package-info.java deleted file mode 100644 index 520094cd593..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.config.server.restapi.resources; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 24666f42dc9..70db406bd53 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -178,7 +178,7 @@ public class SessionPreparer { void preprocess() { try { - this.applicationPackage = context.getApplicationPackage().preprocess(properties.zone().id(), null, logger); + this.applicationPackage = context.getApplicationPackage().preprocess(properties.zone(), null, logger); } catch (IOException | TransformerException | ParserConfigurationException | SAXException e) { throw new RuntimeException("Error deploying application package", e); } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index fd77fedd789..1a0b61fbe96 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -68,13 +68,6 @@ </components> </rest-api> - <rest-api path="status" jersey2="true"> - <components bundle="configserver"> - <package>com.yahoo.vespa.config.server.restapi.impl</package> - <package>com.yahoo.vespa.config.server.restapi.resources</package> - </components> - </rest-api> - <handler id='com.yahoo.vespa.config.server.http.HttpGetConfigHandler' bundle='configserver'> <binding>http://*/config/v1/*/*</binding> <binding>https://*/config/v1/*/*</binding> @@ -91,6 +84,9 @@ <binding>http://*/config/v1/*/*/</binding> <binding>https://*/config/v1/*/*/</binding> </handler> + <handler id='com.yahoo.vespa.config.server.http.status.StatusHandler' bundle='configserver'> + <binding>http://*/status</binding> + </handler> <handler id='com.yahoo.vespa.config.server.http.v2.TenantHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/</binding> <binding>https://*/application/v2/tenant/</binding> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 71f4e4add50..b19d6e2e257 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -44,13 +44,9 @@ public class HttpGetConfigHandlerTest { mockRequestHandler.setAllConfigs(new HashSet<ConfigKey<?>>() {{ add(new ConfigKey<>("bar", "myid", "foo")); }} ); - handler = new HttpGetConfigHandler(new Executor() { - @SuppressWarnings("NullableProblems") - @Override - public void execute(Runnable command) { - command.run(); - } - }, mockRequestHandler, AccessLog.voidAccessLog()); + handler = new HttpGetConfigHandler( + HttpGetConfigHandler.testOnlyContext(), + mockRequestHandler); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java index 76844bb7c21..bf881e7a546 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java @@ -25,7 +25,7 @@ public class HttpHandlerTest { @Test public void testResponse() throws IOException { final String message = "failed"; - HttpHandler httpHandler = new HttpTestHandler(Executors.newSingleThreadExecutor(), AccessLog.voidAccessLog(), new InvalidApplicationException(message)); + HttpHandler httpHandler = new HttpTestHandler(new InvalidApplicationException(message)); HttpResponse response = httpHandler.handle(HttpRequest.createTestRequest("foo", com.yahoo.jdisc.http.HttpRequest.Method.GET)); assertThat(response.getStatus(), is(Response.Status.BAD_REQUEST)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -38,8 +38,8 @@ public class HttpHandlerTest { private static class HttpTestHandler extends HttpHandler { private RuntimeException exception; - public HttpTestHandler(Executor executor, AccessLog accessLog, RuntimeException exception) { - super(executor, accessLog); + public HttpTestHandler(RuntimeException exception) { + super(HttpHandler.testOnlyContext()); this.exception = exception; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index db8526150bf..01618e5a85f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -37,18 +37,9 @@ public class HttpListConfigsHandlerTest { mockRequestHandler.setAllConfigs(new HashSet<ConfigKey<?>>() {{ add(new ConfigKey<>("bar", "conf/id/", "foo")); }} ); - handler = new HttpListConfigsHandler(new Executor() { - @Override - public void execute(Runnable command) { - command.run(); - } - }, AccessLog.voidAccessLog(), mockRequestHandler); - namedHandler = new HttpListNamedConfigsHandler(new Executor() { - @Override - public void execute(Runnable command) { - command.run(); - } - }, mockRequestHandler, AccessLog.voidAccessLog()); + HttpListConfigsHandler.Context ctx = HttpListConfigsHandler.testOnlyContext(); + handler = new HttpListConfigsHandler(ctx, mockRequestHandler); + namedHandler = new HttpListNamedConfigsHandler(ctx, mockRequestHandler); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java index 7aff8f9410b..b6d9ab5d618 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java @@ -54,7 +54,7 @@ public class SessionExampleHandlerTest { public static class SessionExampleHandler extends ThreadedHttpRequestHandler { public SessionExampleHandler(Executor executor) { - super(executor); + super(executor, null); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/status/StatusHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/status/StatusHandlerTest.java new file mode 100644 index 00000000000..52e9591e63c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/status/StatusHandlerTest.java @@ -0,0 +1,40 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.status; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static org.junit.Assert.assertEquals; + +/** + * @author hmusum + */ +public class StatusHandlerTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + @Test + public void require_that_handler_works() throws IOException { + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); + StatusHandler handler = new StatusHandler(StatusHandler.testOnlyContext(), componentRegistry); + + HttpResponse response = handler.handle(HttpRequest.createTestRequest("/status", GET)); + JsonNode jsonNode = mapper.readTree(SessionHandlerTest.getRenderedString(response)); + + ConfigserverConfig expectedConfig = componentRegistry.getConfigserverConfig(); + assertEquals(expectedConfig.rpcport(), jsonNode.get("configserverConfig").get("rpcport").asInt()); + assertEquals(expectedConfig.applicationDirectory(), jsonNode.get("configserverConfig").get("applicationDirectory").asText()); + + assertEquals(1, jsonNode.get("modelVersions").size()); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index a17d485a425..c34dbe76a43 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -52,8 +52,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { testTenantBuilder.tenants().get(tenant2).getLocalSessionRepo().addSession(new MockSession(3l, FilesApplicationPackage.fromFile(new File("src/test/apps/content2")))); testTenantBuilder.tenants().get(tenant1).getApplicationRepo().createPutApplicationTransaction(idTenant1, 2l).commit(); testTenantBuilder.tenants().get(tenant2).getApplicationRepo().createPutApplicationTransaction(idTenant2, 3l).commit(); - handler = new ApplicationHandler(Runnable::run, - AccessLog.voidAccessLog(), + handler = new ApplicationHandler(ApplicationHandler.testOnlyContext(), Zone.defaultZone(), new ApplicationRepository(testTenantBuilder.createTenants(), new MockProvisioner(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 5552758a0a6..8ac64e5b28a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -96,7 +96,8 @@ public class ApplicationHandlerTest { mockHttpProxy, new MockLogServerLogGrabber()); listApplicationsHandler = new ListApplicationsHandler( - Runnable::run, AccessLog.voidAccessLog(), tenants, Zone.defaultZone()); + ListApplicationsHandler.testOnlyContext(), + tenants, Zone.defaultZone()); } private ApplicationHandler createMockApplicationHandler( @@ -105,8 +106,7 @@ public class ApplicationHandlerTest { HttpProxy httpProxy, LogServerLogGrabber logServerLogGrabber) { return new ApplicationHandler( - Runnable::run, - AccessLog.voidAccessLog(), + ApplicationHandler.testOnlyContext(), Zone.defaultZone(), new ApplicationRepository(tenants, HostProvisionerProvider.withProvisioner(provisioner), @@ -118,8 +118,7 @@ public class ApplicationHandlerTest { private ApplicationHandler createApplicationHandler(Tenants tenants) { return new ApplicationHandler( - Runnable::run, - AccessLog.voidAccessLog(), + ApplicationHandler.testOnlyContext(), Zone.defaultZone(), new ApplicationRepository(tenants, HostProvisionerProvider.withProvisioner(provisioner), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index e439f424c45..fe25170d8ba 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -52,9 +52,9 @@ public class HostHandlerTest { hostRegistries = testComponentRegistry.getHostRegistries(); hostRegistries.createApplicationHostRegistry(mytenant).update(ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()), Collections.singletonList(hostname)); hostRegistries.getTenantHostRegistry().update(mytenant, Collections.singletonList(hostname)); - hostHandler = new HostHandler(command -> { - command.run(); - }, AccessLog.voidAccessLog(), testComponentRegistry); + hostHandler = new HostHandler( + HostHandler.testOnlyContext(), + testComponentRegistry); return hostHandler; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index cc18e279002..11bacc30b27 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -49,9 +49,9 @@ public class HttpGetConfigHandlerTest { TestTenantBuilder tb = new TestTenantBuilder(); tb.createTenant(tenant).withRequestHandler(mockRequestHandler).build(); Tenants tenants = tb.createTenants(); - handler = new HttpGetConfigHandler(command -> { - command.run(); - }, AccessLog.voidAccessLog(), tenants); + handler = new HttpGetConfigHandler( + HttpGetConfigHandler.testOnlyContext(), + tenants); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index a66e9542a5f..e7ccd9f957e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -45,12 +45,12 @@ public class HttpListConfigsHandlerTest { TestTenantBuilder tb = new TestTenantBuilder(); tb.createTenant(TenantName.from("mytenant")).withRequestHandler(mockRequestHandler).build(); Tenants tenants = tb.createTenants(); - handler = new HttpListConfigsHandler(command -> { - command.run(); - }, AccessLog.voidAccessLog(), tenants, Zone.defaultZone()); - namedHandler = new HttpListNamedConfigsHandler(command -> { - command.run(); - }, AccessLog.voidAccessLog(), tenants, Zone.defaultZone()); + handler = new HttpListConfigsHandler( + HttpListConfigsHandler.testOnlyContext(), + tenants, Zone.defaultZone()); + namedHandler = new HttpListNamedConfigsHandler( + HttpListConfigsHandler.testOnlyContext(), + tenants, Zone.defaultZone()); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java index 9e7853a8fdf..3233d9598d1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -39,10 +39,10 @@ public class ListApplicationsHandlerTest { applicationRepo = testBuilder.tenants().get(mytenant).getApplicationRepo(); applicationRepo2 = testBuilder.tenants().get(foobar).getApplicationRepo(); Tenants tenants = testBuilder.createTenants(); - handler = new ListApplicationsHandler(Runnable::run, - AccessLog.voidAccessLog(), - tenants, - new Zone(Environment.dev, RegionName.from("us-east"))); + handler = new ListApplicationsHandler( + ListApplicationsHandler.testOnlyContext(), + tenants, + new Zone(Environment.dev, RegionName.from("us-east"))); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 6542c865d56..04bc8d7b49a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -373,13 +373,12 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { .withApplicationRepo(applicationRepo) .build(); return new SessionActiveHandler( - Runnable::run, - AccessLog.voidAccessLog(), - testTenantBuilder.createTenants(), - Zone.defaultZone(), + SessionActiveHandler.testOnlyContext(), new ApplicationRepository(testTenantBuilder.createTenants(), hostProvisioner, - Clock.systemUTC())); + Clock.systemUTC()), + testTenantBuilder.createTenants(), + Zone.defaultZone()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 1d831032416..e4841930cc8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -161,15 +161,11 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { private SessionContentHandler createHandler() throws Exception { TestTenantBuilder testTenantBuilder = new TestTenantBuilder(); testTenantBuilder.createTenant(tenant).getLocalSessionRepo().addSession(new MockSession(1l, FilesApplicationPackage.fromFile(createTestApp()))); - return new SessionContentHandler(new Executor() { - @SuppressWarnings("NullableProblems") - @Override - public void execute(Runnable command) { - command.run(); - } - }, AccessLog.voidAccessLog(), testTenantBuilder.createTenants(), - new ApplicationRepository(testTenantBuilder.createTenants(), - new SessionHandlerTest.MockProvisioner(), - Clock.systemUTC())); + return new SessionContentHandler( + SessionContentHandler.testOnlyContext(), + new ApplicationRepository(testTenantBuilder.createTenants(), + new SessionHandlerTest.MockProvisioner(), + Clock.systemUTC()), + testTenantBuilder.createTenants()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 65b12490b17..fc9264a6ef5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -243,10 +243,13 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { private SessionCreateHandler createHandler(Tenants tenants) throws Exception { TestTenantBuilder testTenantBuilder = new TestTenantBuilder(); final ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); - return new SessionCreateHandler(Runnable::run, AccessLog.voidAccessLog(), tenants, configserverConfig, - new ApplicationRepository(testTenantBuilder.createTenants(), - new SessionHandlerTest.MockProvisioner(), - Clock.systemUTC())); + return new SessionCreateHandler( + SessionCreateHandler.testOnlyContext(), + new ApplicationRepository(testTenantBuilder.createTenants(), + new SessionHandlerTest.MockProvisioner(), + Clock.systemUTC()), + tenants, configserverConfig); + } private HttpRequest post() throws FileNotFoundException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 74a2dcf8054..1759cd68062 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -383,10 +383,13 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { private SessionHandler createHandler(TestTenantBuilder builder) { final ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); - return new SessionPrepareHandler(Runnable::run, AccessLog.voidAccessLog(), builder.createTenants(), configserverConfig, - new ApplicationRepository(builder.createTenants(), - new MockProvisioner(), - Clock.systemUTC())); + return new SessionPrepareHandler( + SessionPrepareHandler.testOnlyContext(), + new ApplicationRepository(builder.createTenants(), + new MockProvisioner(), + Clock.systemUTC()), + builder.createTenants(), configserverConfig); + } private TestTenantBuilder addTenant(TenantName tenantName, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index ce4b25fe529..e948bf68970 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -27,7 +27,9 @@ public class TenantHandlerTest extends TenantTest { @Before public void setup() throws Exception { - handler = new TenantHandler(testExecutor(), null, tenants); + handler = new TenantHandler( + TenantHandler.testOnlyContext(), + tenants); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/restapi/impl/StatusResourceTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/restapi/impl/StatusResourceTest.java deleted file mode 100644 index b912b67c7db..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/restapi/impl/StatusResourceTest.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.restapi.impl; - -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.restapi.resources.StatusInformation; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author lulf - * @since 5.1 - */ -public class StatusResourceTest { - @Test - public void require_that_status_handler_responds_to_ping() throws IOException { - StatusResource handler = new StatusResource(null, null, null, null, null, null, null, new TestComponentRegistry.Builder().build()); - assertNotNull(handler.getStatus().configserverConfig); - } - - @Test - public void require_that_generated_config_is_converted() { - ConfigserverConfig orig = new ConfigserverConfig(new ConfigserverConfig.Builder()); - StatusInformation.ConfigserverConfig conv = new StatusInformation.ConfigserverConfig(orig); - assertThat(conv.applicationDirectory, is(getDefaults().underVespaHome(orig.applicationDirectory()))); - assertThat(conv.configModelPluginDir.size(), is(orig.configModelPluginDir().size())); - assertThat(conv.zookeeeperserver.size(), is(orig.zookeeperserver().size())); - assertThat(conv.zookeeperBarrierTimeout, is(orig.zookeeper().barrierTimeout())); - assertThat(conv.configServerDBDir, is(getDefaults().underVespaHome(orig.configServerDBDir()))); - assertThat(conv.masterGeneration, is(orig.masterGeneration())); - assertThat(conv.maxgetconfigclients, is(orig.maxgetconfigclients())); - assertThat(conv.multitenant, is(orig.multitenant())); - assertThat(conv.numDelayedResponseThreads, is(orig.numDelayedResponseThreads())); - assertThat(conv.numthreads, is(orig.numthreads())); - assertThat(conv.payloadCompressionType, is(orig.payloadCompressionType())); - assertThat(conv.rpcport, is(orig.rpcport())); - assertThat(conv.sessionLifetime, is(orig.sessionLifetime())); - assertThat(conv.zookeepercfg, is(getDefaults().underVespaHome(orig.zookeepercfg()))); - } -} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 0095fcece4f..4f365ebbab3 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -35,7 +35,42 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { this(executor, accessLog, null); } + public static class Context { + final Executor executor; + final AccessLog accessLog; + final Metric metric; + @Inject + public Context(Executor executor, AccessLog accessLog, Metric metric) { + this.executor = executor; + this.accessLog = accessLog; + this.metric = metric; + } + public Context(Context other) { + this.executor = other.executor; + this.accessLog = other.accessLog; + this.metric = other.metric; + } + } + public static Context testOnlyContext() { + return new Context(new Executor() { + @Override + public void execute(Runnable command) { + command.run(); + } + }, + AccessLog.voidAccessLog(), + null); + } + @Inject + public LoggingRequestHandler(Context ctx) { + this(ctx.executor, ctx.accessLog, ctx.metric); + } + + public LoggingRequestHandler(Context ctx, boolean allowAsyncResponse) { + this(ctx.executor, ctx.accessLog, ctx.metric, allowAsyncResponse); + } + public LoggingRequestHandler(Executor executor, AccessLog accessLog, Metric metric) { this(executor, accessLog, metric, false); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index fa2ee8e89a9..bf696771b20 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -29,6 +29,7 @@ import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.jrt.ListenFailedException; +import com.yahoo.log.LogLevel; import com.yahoo.log.LogSetup; import com.yahoo.osgi.OsgiImpl; import com.yahoo.vespa.config.ConfigKey; @@ -88,6 +89,7 @@ public final class ConfiguredApplication implements Application { static { LogSetup.initVespaLogging("Container"); + log.log(LogLevel.INFO, "Starting container"); } /** diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index a4eb2449064..b83dd6175e1 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -37,7 +37,6 @@ public class Deconstructor implements ComponentDeconstructor { if (component instanceof AbstractComponent) { AbstractComponent abstractComponent = (AbstractComponent) component; if (abstractComponent.isDeconstructable()) { - log.info("Scheduling deconstruction of " + abstractComponent); executor.schedule(new DestructComponentTask(abstractComponent), delay, TimeUnit.SECONDS); } } else if (component instanceof Provider) { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java index 15a8a670a2e..8091397237d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java @@ -25,7 +25,7 @@ import static com.yahoo.prelude.querytransform.NormalizingSearcher.ACCENT_REMOVA * Check sorting specification makes sense to the search cluster before * passing it on to the backend. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ @Before(PhaseNames.BACKEND) @After(ACCENT_REMOVAL) @@ -118,6 +118,7 @@ public class ValidateSortingSearcher extends Searcher { for (Sorting.FieldOrder f : l) { String name = f.getFieldName(); if ("[rank]".equals(name) || "[docid]".equals(name)) { + // built-in constants - ok } else if (names.containsKey(name)) { AttributesConfig.Attribute attrConfig = names.get(name); if (attrConfig != null) { @@ -166,18 +167,13 @@ public class ValidateSortingSearcher extends Searcher { locale = "en_US"; } - // getLogger().info("locale = " + locale + " attrConfig.sortlocale.value() = " + attrConfig.sortlocale.value() + " query.getLanguage() = " + query.getModel().getLanguage()); - // getLogger().info("locale = " + locale); - Sorting.UcaSorter.Strength strength = sorter.getStrength(); if (sorter.getStrength() == Sorting.UcaSorter.Strength.UNDEFINED) { strength = config2Strength(attrConfig.sortstrength()); } if ((sorter.getStrength() == Sorting.UcaSorter.Strength.UNDEFINED) || (sorter.getLocale() == null) || sorter.getLocale().isEmpty()) { - // getLogger().info("locale = " + locale + " strength = " + strength.toString()); sorter.setLocale(locale, strength); } - //getLogger().info("locale = " + locale + " strength = " + strength.toString() + "decompose = " + sorter.getDecomposition()); } } else { return ErrorMessage.createInvalidQueryParameter("The cluster " + getClusterName() + " has attribute config for field: " + name); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java index f1c584c7a3c..04dd91670c1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.identifiers; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import java.util.Objects; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java index d3e1b881bbd..20e9710f092 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import java.util.Map; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentityVerifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentityVerifier.java index bfaa6c2acda..527efaab946 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentityVerifier.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzIdentityVerifier.java @@ -1,21 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.athenz; +import org.apache.http.conn.ssl.X509HostnameVerifier; + import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLException; import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocket; import java.security.cert.X509Certificate; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; /** - * A {@link HostnameVerifier} that validates Athenz x509 certificates using the identity in the Common Name attribute. + * A {@link HostnameVerifier} / {@link X509HostnameVerifier} that validates + * Athenz x509 certificates using the identity in the Common Name attribute. * * @author bjorncs */ // TODO Move to dedicated Athenz bundle -public class AthenzIdentityVerifier implements HostnameVerifier { +public class AthenzIdentityVerifier implements X509HostnameVerifier { private static final Logger log = Logger.getLogger(AthenzIdentityVerifier.class.getName()); @@ -29,13 +34,36 @@ public class AthenzIdentityVerifier implements HostnameVerifier { public boolean verify(String hostname, SSLSession session) { try { X509Certificate cert = (X509Certificate) session.getPeerCertificates()[0]; - AthenzIdentity certificateIdentity = AthenzUtils.createAthenzIdentity(cert); - return allowedIdentities.contains(certificateIdentity); + return isTrusted(AthenzUtils.createAthenzIdentity(cert)); } catch (SSLPeerUnverifiedException e) { log.log(Level.WARNING, "Unverified client: " + hostname); return false; } } + @Override + public void verify(String host, SSLSocket ssl) { + // all sockets allowed + } + + @Override + public void verify(String hostname, X509Certificate certificate) throws SSLException { + AthenzIdentity identity = AthenzUtils.createAthenzIdentity(certificate); + if (!isTrusted(identity)) { + throw new SSLException("Athenz identity is not trusted: " + identity.getFullName()); + } + } + + @Override + public void verify(String hostname, String[] cns, String[] subjectAlts) throws SSLException { + AthenzIdentity identity = AthenzUtils.createAthenzIdentity(cns[0]); + if (!isTrusted(identity)) { + throw new SSLException("Athenz identity is not trusted: " + identity.getFullName()); + } + } + + private boolean isTrusted(AthenzIdentity identity) { + return allowedIdentities.contains(identity); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java new file mode 100644 index 00000000000..f718b86ca40 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java @@ -0,0 +1,22 @@ +package com.yahoo.vespa.hosted.controller.api.integration.zone; + +/** + * A ZoneId list which can be filtered in various ways; elements can be accessed after at least one filter. + * + * The methods here return instances of {@link ZoneList}, which extends ZoneFilter, but with accessors and additional filters. + * This forces the developer to consider which of the filters in this class to apply, prior to processing any zones. + * + * @author jvenstad + */ +public interface ZoneFilter { + + /** Negates the next filter. */ + ZoneFilter not(); + + /** All zones from the initial pool. */ + ZoneList all(); + + /** Zones where which are managed by the controller. */ + ZoneList controllerManaged(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java new file mode 100644 index 00000000000..e68bf0ccc24 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java @@ -0,0 +1,72 @@ +package com.yahoo.vespa.hosted.controller.api.integration.zone; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * A Zones.List implementation which assumes all zones are controllerManaged. + * + * @author jvenstad + */ +public class ZoneFilterMock implements ZoneList { + + private final java.util.List<ZoneId> zones; + private final boolean negate; + + private ZoneFilterMock(java.util.List<ZoneId> zones, boolean negate) { + this.negate = negate; + this.zones = zones; + } + + public static ZoneFilter from(Collection<ZoneId> zones) { + return new ZoneFilterMock(new ArrayList<>(zones), false); + } + + @Override + public ZoneList not() { + return new ZoneFilterMock(zones, ! negate); + } + + @Override + public ZoneList all() { + return filter(zoneId -> true); + } + + @Override + public ZoneList controllerManaged() { + return all(); + } + + @Override + public ZoneList in(Environment environment) { + return filter(zoneId -> zoneId.environment() == environment); + } + + @Override + public ZoneList in(RegionName region) { + return filter(zoneId -> zoneId.region().equals(region)); + } + + @Override + public ZoneList zones(ZoneId... zones) { + return filter(zoneId -> new HashSet<>(Arrays.asList(zones)).contains(zoneId)); + } + + @Override + public java.util.List<ZoneId> ids() { + return Collections.unmodifiableList(zones); + } + + private ZoneFilterMock filter(Predicate<ZoneId> condition) { + return new ZoneFilterMock(zones.stream().filter(negate ? condition.negate() : condition).collect(Collectors.toList()), false); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java index d51a8d5e0c9..21ac7a654b8 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ZoneId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java @@ -1,5 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; +package com.yahoo.vespa.hosted.controller.api.integration.zone; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import java.util.Objects; @@ -11,7 +14,7 @@ import java.util.Objects; * @author jvenstad */ public class ZoneId { - // TODO: Replace usages of zone + region with usages of this. + // TODO: Replace usages of environment + region with usages of this. private final Environment environment; private final RegionName region; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java new file mode 100644 index 00000000000..cd263769864 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java @@ -0,0 +1,34 @@ +package com.yahoo.vespa.hosted.controller.api.integration.zone; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; + +import java.util.List; + +/** + * Provides filters for and access to a list of ZoneIds. + * + * This is typically offered after an initial filter from {@link ZoneFilter} has been applied. + * This forces the developer to consider which zones to process. + * + * @author jvenstad + */ +public interface ZoneList extends ZoneFilter { + + /** Negates the next filter. */ + @Override + ZoneList not(); + + /** Zones in the given environment. */ + ZoneList in(Environment environment); + + /** Zones in the given region. */ + ZoneList in(RegionName region); + + /** Only the given zones — combine with not() for best effect! */ + ZoneList zones(ZoneId... zones); + + /** Returns the id of all zones in this list as — you guessed it — a list. */ + List<ZoneId> ids(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index af7c464b8d7..e95c297b593 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.zone; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; import java.net.URI; import java.time.Duration; @@ -24,7 +23,7 @@ public interface ZoneRegistry { boolean hasZone(ZoneId zoneId); /** Returns a list containing the id of all zones in this registry. */ - List<ZoneId> zones(); + ZoneFilter zones(); /** Returns the default region for the given environment, if one is configured. */ Optional<RegionName> getDefaultRegion(Environment environment); @@ -39,7 +38,7 @@ public interface ZoneRegistry { /** Returns a list with all known config servers in the given zone, with a secure connection URL. */ List<URI> getConfigServerSecureUris(ZoneId zoneId); - /** Returns a URL with the logs for the given deployment, if loggin is configured for its zone. */ + /** Returns a URL with the logs for the given deployment, if logging is configured for its zone. */ Optional<URI> getLogServerUri(DeploymentId deploymentId); /** Returns the time to live for deployments in the given zone, or empty if this is infinite. */ @@ -54,4 +53,7 @@ public interface ZoneRegistry { /** Returns the system of this registry. */ SystemName system(); + /** Return the configserver's Athenz service identity */ + AthenzService getConfigserverAthenzService(ZoneId zoneId); + } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java index aa3d1be879e..0ba607a235b 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.identifiers; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index ae2de96f511..b75f80917a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -7,7 +7,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index ec1051a3674..28fa311b841 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -7,7 +7,7 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.InstanceEndpoints; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index f50958f0e66..44e4cf0740f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -7,12 +7,13 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; @@ -23,7 +24,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingSe import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; @@ -156,9 +156,17 @@ public class Controller extends AbstractComponent { return zoneRegistry.getLogServerUri(deploymentId); } + /** + * @deprecated Use {@link #getSecureConfigServerUris(ZoneId)} instead + */ + @Deprecated public List<URI> getConfigServerUris(ZoneId zoneId) { return zoneRegistry.getConfigServerUris(zoneId); } + + public List<URI> getSecureConfigServerUris(ZoneId zoneId) { + return zoneRegistry.getConfigServerSecureUris(zoneId); + } public ZoneRegistry zoneRegistry() { return zoneRegistry; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 72ed1a42435..5fa5b8c318b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -5,8 +5,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Zone; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 4d1a009806f..07d51b2b9c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -121,9 +121,9 @@ public class ApplicationList { return listOf(list.stream().filter(a -> !a.productionDeployments().isEmpty())); } - /** Returns the subset of applications which started failing after the given instant */ - public ApplicationList startedFailingOnVersionAfter(Version version, Instant instant) { - return listOf(list.stream().filter(application -> JobList.from(application).firstFailing().on(version).firstFailing().after(instant).anyMatch())); + /** Returns the subset of applications which started failing on the given version */ + public ApplicationList startedFailingOn(Version version) { + return listOf(list.stream().filter(application -> ! JobList.from(application).firstFailing().on(version).isEmpty())); } /** Returns the subset of applications which has the given upgrade policy */ @@ -209,32 +209,32 @@ public class ApplicationList { } private static boolean failingOn(Version version, Application application) { - return JobList.from(application) + return ! JobList.from(application) .failing() .lastCompleted().on(version) - .anyMatch(); + .isEmpty(); } private static boolean currentlyUpgrading(Change.VersionChange change, Application application, Instant jobTimeoutLimit) { - return JobList.from(application) + return ! JobList.from(application) .running(jobTimeoutLimit) .lastTriggered().on(change.version()) - .anyMatch(); + .isEmpty(); } private static boolean failingUpgradeToVersionSince(Application application, Version version, Instant threshold) { - return JobList.from(application) + return ! JobList.from(application) .not().failingApplicationChange() .firstFailing().before(threshold) .lastCompleted().on(version) - .anyMatch(); + .isEmpty(); } private static boolean failingApplicationChangeSince(Application application, Instant threshold) { - return JobList.from(application) + return ! JobList.from(application) .failingApplicationChange() .firstFailing().before(threshold) - .anyMatch(); + .isEmpty(); } /** Convenience converter from a stream to an ApplicationList */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index b9d07249cb2..2364e87b345 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -3,8 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.config.provision.ClusterSpec.Id; -import com.yahoo.config.provision.Zone; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import java.time.Instant; import java.util.HashMap; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index ec8b2d6d019..eea94411109 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -101,12 +101,12 @@ public class DeploymentJobs { /** Returns whether this has some job status which is not a success */ public boolean hasFailures() { - return JobList.from(status.values()).failing().anyMatch(); + return ! JobList.from(status.values()).failing().isEmpty(); } /** Returns whether any job is currently in progress */ public boolean isRunning(Instant timeoutLimit) { - return JobList.from(status.values()).running(timeoutLimit).anyMatch(); + return ! JobList.from(status.values()).running(timeoutLimit).isEmpty(); } /** Returns whether the given job type is currently running and was started after timeoutLimit */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java index 6223b07d27a..161035b1164 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -57,8 +57,6 @@ public class JobList { public boolean isEmpty() { return list.isEmpty(); } - public boolean anyMatch() { return ! isEmpty(); } - public int size() { return list.size(); } // ----------------------------------- Basic filters diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java index 2bf64571bdf..dd7befb6d63 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedApplication; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 192901165be..90237a17fb9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -344,10 +344,10 @@ public class DeploymentTrigger { } private boolean isRunningProductionJob(Application application) { - return JobList.from(application) + return ! JobList.from(application) .production() .running(jobTimeoutLimit()) - .anyMatch(); + .isEmpty(); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index ad7fa90967b..cf0600f87bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.Zone; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -18,7 +16,6 @@ import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index 58e32344372..b889179750e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 324868878af..e30ccbe7950 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -75,17 +76,21 @@ public class DeploymentIssueReporter extends Maintainer { * longer than the set grace period, or update this list if the issue already exists. */ private void maintainPlatformIssue(List<Application> applications) { - if ( ! (controller().versionStatus().version(controller().systemVersion()).confidence() == broken)) + Version systemVersion = controller().systemVersion(); + + if ((controller().versionStatus().version(systemVersion).confidence() != broken)) + return; + + if (ApplicationList.from(applications) + .failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)) + .isEmpty()) return; List<ApplicationId> failingApplications = ApplicationList.from(applications) - .failingUpgradeToVersionSince(controller().systemVersion(), controller().clock().instant().minus(upgradeGracePeriod)) - .asList().stream() - .map(Application::id) - .collect(Collectors.toList()); + .failingUpgradeToVersionSince(systemVersion, controller().clock().instant()) + .idList(); - if ( ! failingApplications.isEmpty()) - deploymentIssues.fileUnlessOpen(failingApplications, controller().systemVersion()); + deploymentIssues.fileUnlessOpen(failingApplications, systemVersion); } private Tenant ownerOf(ApplicationId applicationId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 762f12c3e8a..9c77ebc4bc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -6,9 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index e67b96c22ad..7d06bbde081 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -3,11 +3,14 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentityVerifier; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzSslContextProvider; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.apache.http.Header; import org.apache.http.client.config.RequestConfig; @@ -34,8 +37,11 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import static java.util.Collections.singleton; + /** * @author Haakon Dybdahl + * @author bjorncs */ @SuppressWarnings("unused") // Injected public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { @@ -43,9 +49,13 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { private static final Duration PROXY_REQUEST_TIMEOUT = Duration.ofSeconds(10); private final ZoneRegistry zoneRegistry; + private final AthenzSslContextProvider sslContextProvider; - public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry) { + @Inject + public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, + AthenzSslContextProvider sslContextProvider) { this.zoneRegistry = zoneRegistry; + this.sslContextProvider = sslContextProvider; } @Override @@ -57,10 +67,10 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { ZoneId zoneId = ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion()); // Make a local copy of the list as we want to manipulate it in case of ping problems. - List<URI> allServers = new ArrayList<>(zoneRegistry.getConfigServerUris(zoneId)); + List<URI> allServers = new ArrayList<>(zoneRegistry.getConfigServerSecureUris(zoneId)); StringBuilder errorBuilder = new StringBuilder(); - if (queueFirstServerIfDown(allServers)) { + if (queueFirstServerIfDown(allServers, proxyRequest)) { errorBuilder.append("Change ordering due to failed ping."); } for (URI uri : allServers) { @@ -81,15 +91,15 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { private ProxyResponse createDiscoveryResponse(ProxyRequest proxyRequest) { ObjectMapper mapper = new ObjectMapper(); DiscoveryResponseStructure responseStructure = new DiscoveryResponseStructure(); + String environmentName = proxyRequest.getEnvironment(); - List<ZoneId> zones = zoneRegistry.zones(); - for (ZoneId zone : zones) { - if (!"".equals(proxyRequest.getEnvironment()) && - !proxyRequest.getEnvironment().equals(zone.environment().value())) { - continue; - } + ZoneList zones = zoneRegistry.zones().all(); + if ( ! environmentName.isEmpty()) + zones = zones.in(Environment.from(environmentName)); + + for (ZoneId zoneId : zones.ids()) { responseStructure.uris.add(proxyRequest.getScheme() + "://" + proxyRequest.getControllerPrefix() + - zone.environment().name() + "/" + zone.region().value()); + zoneId.environment().name() + "/" + zoneId.region().value()); } JsonNode node = mapper.valueToTree(responseStructure); return new ProxyResponse(proxyRequest, node.toString(), 200, Optional.empty(), "application/json"); @@ -111,17 +121,17 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { copyHeaders(proxyRequest.getHeaders(), requestBase, new HashSet<>()); RequestConfig config = RequestConfig.custom() - .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); + .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); try ( - CloseableHttpClient client = createHttpClient(config); + CloseableHttpClient client = createHttpClient(config, sslContextProvider, zoneRegistry, proxyRequest); CloseableHttpResponse response = client.execute(requestBase); ) { if (response.getStatusLine().getStatusCode() / 100 == 5) { errorBuilder.append("Talking to server ").append(uri.getHost()); errorBuilder.append(", got ").append(response.getStatusLine().getStatusCode()).append(" ") - .append(streamToString(response.getEntity().getContent())).append("\n"); + .append(streamToString(response.getEntity().getContent())).append("\n"); return Optional.empty(); } final Header contentHeader = response.getLastHeader("Content-Type"); @@ -202,7 +212,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { * if it is not responding, we try the other servers first. False positive/negatives are not critical, * but will increase latency to some extent. */ - private boolean queueFirstServerIfDown(List<URI> allServers) { + private boolean queueFirstServerIfDown(List<URI> allServers, ProxyRequest proxyRequest) { if (allServers.size() < 2) { return false; } @@ -215,7 +225,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { .setConnectionRequestTimeout(timeout) .setSocketTimeout(timeout).build(); try ( - CloseableHttpClient client = createHttpClient(config); + CloseableHttpClient client = createHttpClient(config, sslContextProvider, zoneRegistry, proxyRequest); CloseableHttpResponse response = client.execute(httpget); ) { @@ -232,9 +242,19 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { return true; } - private static CloseableHttpClient createHttpClient(RequestConfig config) { + private static CloseableHttpClient createHttpClient(RequestConfig config, + AthenzSslContextProvider sslContextProvider, + ZoneRegistry zoneRegistry, + ProxyRequest proxyRequest) { + AthenzIdentityVerifier hostnameVerifier = + new AthenzIdentityVerifier( + singleton( + zoneRegistry.getConfigserverAthenzService( + ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion())))); return HttpClientBuilder.create() .setUserAgent("config-server-client") + .setSslcontext(sslContextProvider.get()) + .setHostnameVerifier(hostnameVerifier) .setDefaultRequestConfig(config) .build(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d15686077c6..a7d072d1dae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java index 0b0a2c3ad52..2429565350c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index 83b725ae4c4..282dd79b317 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -69,7 +69,7 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse root(HttpRequest request) { - List<Environment> environments = zoneRegistry.zones().stream() + List<Environment> environments = zoneRegistry.zones().all().ids().stream() .map(ZoneId::environment) .distinct() .sorted(Comparator.comparing(Environment::value)) @@ -89,9 +89,7 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse environment(HttpRequest request, Environment environment) { - List<ZoneId> zones = zoneRegistry.zones().stream() - .filter(zone -> zone.environment() == environment) - .collect(Collectors.toList()); + List<ZoneId> zones = zoneRegistry.zones().all().in(environment).ids(); Slime slime = new Slime(); Cursor root = slime.setArray(); zones.forEach(zone -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 772dd1f6cb1..68dc2325687 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v2; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -20,7 +18,6 @@ import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.yolean.Exceptions; import java.io.IOException; -import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; @@ -94,16 +91,16 @@ public class ZoneApiHandler extends LoggingRequestHandler { Slime slime = new Slime(); Cursor root = slime.setObject(); Cursor uris = root.setArray("uris"); - zoneRegistry.zones().forEach(zone -> uris.addString(request.getUri() + zoneRegistry.zones().all().ids().forEach(zoneId -> uris.addString(request.getUri() .resolve("/zone/v2/") - .resolve(zone.environment().value() + "/") - .resolve(zone.region().value()) + .resolve(zoneId.environment().value() + "/") + .resolve(zoneId.region().value()) .toString())); Cursor zones = root.setArray("zones"); - zoneRegistry.zones().forEach(zone -> { + zoneRegistry.zones().all().ids().forEach(zoneId -> { Cursor object = zones.addObject(); - object.setString("environment", zone.environment().value()); - object.setString("region", zone.region().value()); + object.setString("environment", zoneId.environment().value()); + object.setString("region", zoneId.region().value()); }); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 876bd5fe029..13eec52b97a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -119,10 +119,10 @@ public class VersionStatus { } private static ListMap<Version, String> findConfigServerVersions(Controller controller) { - List<URI> configServers = controller.zoneRegistry().zones().stream() - // TODO: Filter properly. - .filter(zone -> ! zone.region().equals(RegionName.from("us-east-2a"))) - .flatMap(zone -> controller.getConfigServerUris(zone).stream()) + List<URI> configServers = controller.zoneRegistry().zones() + .controllerManaged() + .ids().stream() + .flatMap(zoneId -> controller.getSecureConfigServerUris(zoneId).stream()) .collect(Collectors.toList()); ListMap<Version, String> versions = new ListMap<>(); @@ -184,11 +184,11 @@ public class VersionStatus { VespaVersion.Confidence confidence; // Always compute confidence for system version if (isSystemVersion) { - confidence = VespaVersion.confidenceFrom(statistics, controller, releasedAt); + confidence = VespaVersion.confidenceFrom(statistics, controller); } else { // Keep existing confidence for non-system versions if already computed confidence = confidenceFor(statistics.version(), controller) - .orElse(VespaVersion.confidenceFrom(statistics, controller, releasedAt)); + .orElse(VespaVersion.confidenceFrom(statistics, controller)); } return new VespaVersion(statistics, gitSha.sha, releasedAt, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 4bcee5782ee..ea89a70543c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -42,8 +42,7 @@ public class VespaVersion implements Comparable<VespaVersion> { this.confidence = confidence; } - public static Confidence confidenceFrom(DeploymentStatistics statistics, Controller controller, - Instant releasedAt) { + public static Confidence confidenceFrom(DeploymentStatistics statistics, Controller controller) { // 'production on this': All deployment jobs upgrading to this version have completed without failure ApplicationList productionOnThis = ApplicationList.from(statistics.production(), controller.applications()) .notUpgradingTo(statistics.version()) @@ -58,7 +57,7 @@ public class VespaVersion implements Comparable<VespaVersion> { return Confidence.broken; // 'broken' if 4 non-canary was broken by this, and that is at least 10% of all - if (nonCanaryApplicationsBroken(statistics.version(), failingOnThis, productionOnThis, releasedAt, controller.curator())) + if (nonCanaryApplicationsBroken(statistics.version(), failingOnThis, productionOnThis, controller.curator())) return Confidence.broken; // 'low' unless all canary applications are upgraded @@ -145,9 +144,8 @@ public class VespaVersion implements Comparable<VespaVersion> { private static boolean nonCanaryApplicationsBroken(Version version, ApplicationList failingOnThis, ApplicationList productionOnThis, - Instant releasedAt, CuratorDb curator) { - ApplicationList failingNonCanaries = failingOnThis.without(UpgradePolicy.canary).startedFailingOnVersionAfter(version, releasedAt); + ApplicationList failingNonCanaries = failingOnThis.without(UpgradePolicy.canary).startedFailingOn(version); ApplicationList productionNonCanaries = productionOnThis.without(UpgradePolicy.canary); if (productionNonCanaries.size() + failingNonCanaries.size() == 0 || curator.readIgnoreConfidence()) return false; @@ -156,4 +154,5 @@ public class VespaVersion implements Comparable<VespaVersion> { int brokenByThisVersion = failingNonCanaries.size(); return brokenByThisVersion >= 4 && brokenByThisVersion >= productionOnThis.size() * 0.1; } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 58c74c9d6d2..17801bde546 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 52e1b3ae400..b3ca5491e91 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.Lock; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index 2317b7bc6f1..21072f0b162 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -6,13 +6,15 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneFilter; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneFilterMock; import java.net.URI; import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -63,8 +65,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override - public List<ZoneId> zones() { - return Collections.unmodifiableList(zones); + public ZoneFilter zones() { + return ZoneFilterMock.from(Collections.unmodifiableList(zones)); + } + + public AthenzService getConfigserverAthenzService(ZoneId zone) { + return new AthenzService("vespadomain", "provider-" + zone.environment().value() + "-" + zone.region().value()); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java index a58d2d0fa39..88bbb582564 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import java.util.HashMap; import java.util.Map; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index 47d62f93def..d48f7b84ee6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index 8647b87133e..41892ba666b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index ac282422c89..90721e7be6b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index b281b513f4a..d7389ca94cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -6,9 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index f3fa1e21eda..f252acd44ca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi; import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Zone; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TestIdentities; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java index 4c25bf6fe61..b583aaedde1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index d16a0222e4a..c3ce5c95dcd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -6,7 +6,7 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -85,7 +85,7 @@ public class DeploymentApiTest extends ControllerContainerTest { version.releasedAt(), version.isCurrentSystemVersion(), ImmutableSet.of("config1.test", "config2.test"), - VespaVersion.confidenceFrom(version.statistics(), controller, version.releasedAt()) + VespaVersion.confidenceFrom(version.statistics(), controller) ); censored.add(version); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index 1269bb23105..d680d943f84 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -7,7 +7,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java index 2d92d10b661..6e87304774a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index 9c20c470cf8..782dc6dba4f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -4,7 +4,7 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Request.Method; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.text.Utf8; import com.yahoo.vespa.hosted.controller.ConfigServerProxyMock; import com.yahoo.vespa.hosted.controller.ZoneRegistryMock; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 4f97c078c9b..1c1e2df2c94 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -11,8 +11,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; @@ -21,7 +19,6 @@ import org.junit.Test; import java.net.URI; import java.net.URISyntaxException; import java.time.Duration; -import java.util.Collections; import java.util.List; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; @@ -31,7 +28,6 @@ import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobTy import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -61,7 +57,7 @@ public class VersionStatusTest { public void testSystemVersionIsVersionOfOldestConfigServer() throws URISyntaxException { ControllerTester tester = new ControllerTester(); Version oldest = new Version(5); - tester.configServer().versions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest); + tester.configServer().versions().put(new URI("https://cfg.prod.corp-us-east-1.test:4443"), oldest); VersionStatus versionStatus = VersionStatus.compute(tester.controller()); assertEquals(oldest, versionStatus.systemVersion().get().versionNumber()); } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java index 2bf3f0f8d84..bc94c39d135 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java @@ -12,6 +12,11 @@ import java.util.Optional; * and to avoid OSGi exporting those classes. */ public interface Docker { + /** + * Must be called before any other method. May be called more than once. + */ + void start(); + interface CreateContainerCommand { CreateContainerCommand withLabel(String name, String value); CreateContainerCommand withEnvironment(String name, String value); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 9de2cae604f..fa093e0b4dc 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -59,48 +59,77 @@ public class DockerImpl implements Docker { static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby"; private final int SECONDS_TO_WAIT_BEFORE_KILLING; + private final boolean fallbackTo123OnErrors; private static final String FRAMEWORK_CONTAINER_PREFIX = "/"; + private final DockerConfig config; + private final boolean inProduction; private Optional<DockerImageGarbageCollector> dockerImageGC = Optional.empty(); private CounterWrapper numberOfDockerDaemonFails; + private boolean started = false; private final Object monitor = new Object(); @GuardedBy("monitor") private final Set<DockerImage> scheduledPulls = new HashSet<>(); // Exposed for testing. - final DockerClient dockerClient; + DockerClient dockerClient; + + @Inject + public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) { + this(config, + true, /* fallback to 1.23 on errors */ + metricReceiver, + !config.isRunningLocally()); + } + + private DockerImpl(final DockerConfig config, + boolean fallbackTo123OnErrors, + MetricReceiverWrapper metricReceiverWrapper, + boolean inProduction) { + this.config = config; + this.fallbackTo123OnErrors = fallbackTo123OnErrors; + this.inProduction = inProduction; + if (config == null) { + this.SECONDS_TO_WAIT_BEFORE_KILLING = 10; + } else { + SECONDS_TO_WAIT_BEFORE_KILLING = config.secondsToWaitBeforeKillingContainer(); + } + if (metricReceiverWrapper != null) { + setMetrics(metricReceiverWrapper); + } + } // For testing DockerImpl(final DockerClient dockerClient) { + this(null, false, null, false); this.dockerClient = dockerClient; - this.SECONDS_TO_WAIT_BEFORE_KILLING = 10; } - DockerImpl( - final DockerConfig config, - boolean fallbackTo123OnErrors, - MetricReceiverWrapper metricReceiverWrapper) { - SECONDS_TO_WAIT_BEFORE_KILLING = config.secondsToWaitBeforeKillingContainer(); - - dockerClient = initDockerConnection(config, fallbackTo123OnErrors); - setMetrics(metricReceiverWrapper); + // For testing + DockerImpl(final DockerConfig config, + boolean fallbackTo123OnErrors, + MetricReceiverWrapper metricReceiverWrapper) { + this(config, fallbackTo123OnErrors, metricReceiverWrapper, false); } - @Inject - public DockerImpl(final DockerConfig config, MetricReceiverWrapper metricReceiver) { - this( - config, - true, /* fallback to 1.23 on errors */ - metricReceiver); - - if (!config.isRunningLocally()) { - Duration minAgeToDelete = Duration.ofMinutes(config.imageGCMinTimeToLiveMinutes()); - dockerImageGC = Optional.of(new DockerImageGarbageCollector(minAgeToDelete)); + @Override + public void start() { + if (started) return; + started = true; - try { - setupDockerNetworkIfNeeded(); - } catch (Exception e) { - throw new DockerException("Could not setup docker network", e); + if (config != null) { + if (dockerClient == null) { + dockerClient = initDockerConnection(); + } + if (inProduction) { + Duration minAgeToDelete = Duration.ofMinutes(config.imageGCMinTimeToLiveMinutes()); + dockerImageGC = Optional.of(new DockerImageGarbageCollector(minAgeToDelete)); + + try { + setupDockerNetworkIfNeeded(); + } catch (Exception e) { + throw new DockerException("Could not setup docker network", e); + } } } } @@ -489,7 +518,7 @@ public class DockerImpl implements Docker { } } - private DockerClient initDockerConnection(final DockerConfig config, boolean fallbackTo123orErrors) { + private DockerClient initDockerConnection() { JerseyDockerCmdExecFactory dockerFactory = new JerseyDockerCmdExecFactory() .withMaxPerRouteConnections(config.maxPerRouteConnections()) .withMaxTotalConnections(config.maxTotalConnections()) @@ -508,7 +537,7 @@ public class DockerImpl implements Docker { logger.info("Found version 1.24 or newer of remote API, using 1.23."); } } catch (Exception e) { - if (!fallbackTo123orErrors) { + if (!fallbackTo123OnErrors) { throw e; } logger.log(LogLevel.ERROR, "Failed when trying to figure out remote API version of docker, using 1.23", e); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java index 079d6876043..549af0d85cb 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerTestUtils.java @@ -46,11 +46,13 @@ public class DockerTestUtils { public static DockerImpl getDocker() { if (docker == null) { - docker = new DockerImpl( + DockerImpl tmpDocker = new DockerImpl( dockerConfig, false, /* fallback to 1.23 on errors */ new MetricReceiverWrapper(MetricReceiver.nullImplementation)); - createDockerTestNetworkIfNeeded(docker); + tmpDocker.start(); + createDockerTestNetworkIfNeeded(tmpDocker); + docker = tmpDocker; } return docker; diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 9da20e5a84c..57481666f28 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -956,6 +956,34 @@ TEST_F("ReferenceFieldValue with ID serialization matches Java", RefFixture) { f.verify_cross_language_serialization("reference_with_id", value); } +struct AssociatedDocumentRepoFixture { + const DocumentType& doc_type{repo.getDocumentType()}; + DocumentId doc_id{"doc::testdoc"}; + Document source_doc{doc_type, doc_id}; + + std::unique_ptr<Document> roundtrip_serialize_source_document() { + nbostream stream; + VespaDocumentSerializer serializer(stream); + serializer.write(source_doc); + + auto deserialized_doc = std::make_unique<Document>(); + VespaDocumentDeserializer deserializer(repo, stream, serialization_version); + deserializer.read(*deserialized_doc); + return deserialized_doc; + } +}; + +TEST_F("Deserializing non-empty document associates correct repo with document instance", AssociatedDocumentRepoFixture) { + f.source_doc.setValue(f.doc_type.getField("header field"), IntFieldValue(42)); + auto deserialized_doc = f.roundtrip_serialize_source_document(); + EXPECT_EQUAL(&doc_repo, deserialized_doc->getRepo()); +} + +TEST_F("Deserializing empty document associates correct repo with document instance", AssociatedDocumentRepoFixture) { + auto deserialized_doc = f.roundtrip_serialize_source_document(); + EXPECT_EQUAL(&doc_repo, deserialized_doc->getRepo()); +} + } // namespace TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index 2b45e8a298c..0439d16c25d 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -85,6 +85,7 @@ void VespaDocumentDeserializer::readDocument(Document &value) { Document newDoc(*type, value.getId(), true); value.swap(newDoc); } + value.setRepo(_repo.getDocumentTypeRepo()); FixedTypeRepo repo(_repo.getDocumentTypeRepo(), value.getType()); VarScope<FixedTypeRepo> repo_scope(_repo, repo); diff --git a/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h b/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h index 3ab6834e00a..c977131fcd3 100644 --- a/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h +++ b/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h @@ -90,9 +90,7 @@ public: ~DirectTensorBuilder() {} Tensor::UP build() { - return std::make_unique<SparseTensor>(std::move(_type), - std::move(_cells), - std::move(_stash)); + return std::make_unique<SparseTensor>(std::move(_type), std::move(_cells), std::move(_stash)); } template <class Function> diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h index 09286752550..f74ce257b31 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_builder.h @@ -2,9 +2,8 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vector> #include "sparse_tensor_address_ref.h" +#include <vespa/vespalib/stllike/string.h> namespace vespalib::tensor { @@ -19,16 +18,26 @@ namespace vespalib::tensor { class SparseTensorAddressBuilder { private: - std::vector<char> _address; + vespalib::Array<char> _address; +protected: void append(vespalib::stringref str) { - const char *cstr = str.c_str(); - _address.insert(_address.end(), cstr, cstr + str.size() + 1); + for (size_t i(0); i < str.size() + 1; i++) { + _address.push_back_fast(str[i]); + } + } + void ensure_room(size_t additional) { + if (_address.capacity() < (_address.size() + additional)) { + _address.reserve(_address.size() + additional); + } } public: SparseTensorAddressBuilder() : _address() {} - void add(vespalib::stringref label) { append(label); } - void addUndefined() { _address.emplace_back('\0'); } + void add(vespalib::stringref label) { + ensure_room(label.size()+1); + append(label); + } + void addUndefined() { _address.push_back('\0'); } void clear() { _address.clear(); } SparseTensorAddressRef getAddressRef() const { return SparseTensorAddressRef(&_address[0], _address.size()); diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.cpp index 9693832ea88..e0de63b90d2 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.cpp @@ -47,15 +47,16 @@ TensorAddressCombiner::combine(SparseTensorAddressRef lhsRef, SparseTensorAddressRef rhsRef) { clear(); + ensure_room(lhsRef.size() + rhsRef.size()); SparseTensorAddressDecoder lhs(lhsRef); SparseTensorAddressDecoder rhs(rhsRef); for (auto op : _ops) { switch (op) { case AddressOp::LHS: - add(lhs.decodeLabel()); + append(lhs.decodeLabel()); break; case AddressOp::RHS: - add(rhs.decodeLabel()); + append(rhs.decodeLabel()); break; case AddressOp::BOTH: auto lhsLabel(lhs.decodeLabel()); @@ -63,7 +64,7 @@ TensorAddressCombiner::combine(SparseTensorAddressRef lhsRef, if (lhsLabel != rhsLabel) { return false; } - add(lhsLabel); + append(lhsLabel); } } return true; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h index 8e792cb100a..1a7f2fd8d3c 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h @@ -3,7 +3,8 @@ #pragma once #include "sparse_tensor_address_builder.h" -#include <vespa/eval/tensor/types.h> + +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) namespace vespalib::eval { class ValueType; } namespace vespalib::tensor::sparse { @@ -17,12 +18,11 @@ class TensorAddressCombiner : public SparseTensorAddressBuilder enum class AddressOp { LHS, RHS, BOTH }; std::vector<AddressOp> _ops; - public: TensorAddressCombiner(const eval::ValueType &lhs, const eval::ValueType &rhs); ~TensorAddressCombiner(); - bool combine(SparseTensorAddressRef lhsRef, SparseTensorAddressRef rhsRef); + VESPA_DLL_LOCAL bool combine(SparseTensorAddressRef lhsRef, SparseTensorAddressRef rhsRef); size_t numOverlappingDimensions() const; size_t numDimensions() const { return _ops.size(); } }; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h index ad14da4800b..321690085be 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_ref.h @@ -2,8 +2,8 @@ #pragma once -#include <vector> #include <vespa/vespalib/util/stash.h> +#include <cstring> namespace vespalib { diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp index e89b37ab442..cd5715e7379 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_match.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "sparse_tensor_match.h" +#include <vespa/vespalib/stllike/hash_map.hpp> namespace vespalib::tensor { diff --git a/eval/src/vespa/eval/tensor/tensor_apply.cpp b/eval/src/vespa/eval/tensor/tensor_apply.cpp index 7c518d0516f..8f0610fed65 100644 --- a/eval/src/vespa/eval/tensor/tensor_apply.cpp +++ b/eval/src/vespa/eval/tensor/tensor_apply.cpp @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tensor_apply.h" +#include <vespa/vespalib/stllike/hash_map.hpp> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { template <class TensorT> TensorApply<TensorT>::TensorApply(const TensorImplType &tensor, @@ -17,5 +17,4 @@ TensorApply<TensorT>::TensorApply(const TensorImplType &tensor, template class TensorApply<SparseTensor>; -} // namespace vespalib::tensor -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/tensor/tensor_mapper.cpp b/eval/src/vespa/eval/tensor/tensor_mapper.cpp index 25b369c246d..f1039b08816 100644 --- a/eval/src/vespa/eval/tensor/tensor_mapper.cpp +++ b/eval/src/vespa/eval/tensor/tensor_mapper.cpp @@ -8,6 +8,7 @@ #include "wrapped_simple_tensor.h" #include <vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h> #include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <limits> using vespalib::eval::ValueType; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 70f22296bc1..d9d1b4984eb 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -103,6 +103,7 @@ public class FileReceiver { Files.write(inprogressFile.toPath(), part, StandardOpenOption.WRITE, StandardOpenOption.APPEND); } catch (IOException e) { log.log(LogLevel.ERROR, "Failed writing to file(" + inprogressFile.toPath() + "): " + e.getMessage(), e); + inprogressFile.delete(); throw new RuntimeException("Failed writing to file(" + inprogressFile.toPath() + "): ", e); } currentFileSize += part.length; @@ -247,8 +248,11 @@ public class FileReceiver { log.log(LogLevel.DEBUG, "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); } catch (FileAlreadyExistsException e) { // Don't fail if it already exists (we might get the file from several config servers when retrying, servers are down etc. - // so it might be written already) + // so it might be written already). Delete temp file in that case, to avoid filling the disk. log.log(LogLevel.DEBUG, "File '" + destination.getAbsolutePath() + "' already exists, continuing: " + e.getMessage()); + try { + Files.delete(tempFile.toPath()); + } catch (IOException ioe) { /* ignore failure */} } catch (IOException e) { String message = "Failed moving file '" + tempFile.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'"; log.log(LogLevel.ERROR, message, e); @@ -295,7 +299,7 @@ public class FileReceiver { try { session.addPart(partId, part); } catch (Exception e) { - log.severe("Got exception + " + e); + log.severe("Got exception " + e); retval = 1; } double completeness = (double) session.currentFileSize / (double) session.fileSize; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 031506487a8..d3715a1ff89 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -93,7 +93,7 @@ public class FileReferenceDownloader { downloads.remove(fileReference); download.future().set(Optional.of(file)); } else { - log.log(LogLevel.WARNING, "Received a file " + fileReference + " I did not ask for. Impossible"); + log.log(LogLevel.INFO, "Received '" + fileReference + "', which was not requested. Can be ignored if happening during upgrades/restarts"); } } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java index a78e4f1af40..1291418083b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java @@ -36,12 +36,15 @@ public class StandaloneMain { void run(String bundleLocation) { try { + // We're not logging at this point since the application is responsible + // for setting up logging. System.out.println("debug\tInitializing application without privileges."); loader.init(bundleLocation, false); loader.start(); setupSigTermHandler(); waitForShutdown(); System.out.println("debug\tTrying to shutdown in a controlled manner."); + log.log(Level.INFO, "JDisc shutting down"); loader.stop(); System.out.println("debug\tTrying to clean up in a controlled manner."); loader.destroy(); @@ -50,7 +53,7 @@ public class StandaloneMain { } catch (Throwable e) { System.out.print("debug\tUnexpected: "); e.printStackTrace(); - log.log(Level.SEVERE, "Unexpected: ", e); + log.log(Level.SEVERE, "JDisc exiting: Throwable caught: ", e); System.exit(6); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index 981d4219158..1f2fb40f42f 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -94,7 +94,7 @@ public class ConnectorFactory { private SslConnectionFactory newSslConnectionFactory() { Ssl sslConfig = connectorConfig.ssl(); - SslContextFactory factory = new SslContextFactory(); + SslContextFactory factory = new JDiscSslContextFactory(); sslKeyStoreConfigurator.configure(new DefaultSslKeyStoreContext(factory)); sslTrustStoreConfigurator.configure(new DefaultSslTrustStoreContext(factory)); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscSslContextFactory.java new file mode 100644 index 00000000000..81a6a0c8048 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscSslContextFactory.java @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.security.CertificateUtils; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import java.security.KeyStore; +import java.util.Objects; + +/** + * A modified {@link SslContextFactory} that allows passwordless truststore in combination with password protected keystore. + * + * @author bjorncs + */ +class JDiscSslContextFactory extends SslContextFactory { + + private String trustStorePassword; + + @Override + public void setTrustStorePassword(String password) { + super.setTrustStorePassword(password); + this.trustStorePassword = password; + } + + + // Overriden to stop Jetty from using the keystore password if no truststore password is specified. + @Override + protected KeyStore loadTrustStore(Resource resource) throws Exception { + return CertificateUtils.getKeyStore( + resource != null ? resource : getKeyStoreResource(), + Objects.toString(getTrustStoreType(), getKeyStoreType()), + Objects.toString(getTrustStoreProvider(), getKeyStoreProvider()), + trustStorePassword); + } +} diff --git a/metrics/src/vespa/metrics/metrictimer.h b/metrics/src/vespa/metrics/metrictimer.h index 096ba3e27af..0282c0f17ad 100644 --- a/metrics/src/vespa/metrics/metrictimer.h +++ b/metrics/src/vespa/metrics/metrictimer.h @@ -8,7 +8,7 @@ #pragma once -#include <vespa/metrics/valuemetric.h> +#include "valuemetric.h" #include <chrono> namespace metrics { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index f41d7deb04b..c1f685c78ce 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -9,11 +9,11 @@ import com.yahoo.io.IOUtils; import com.yahoo.net.HostName; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; @@ -50,7 +50,7 @@ public class StorageMaintainer { private static final ObjectMapper objectMapper = new ObjectMapper(); private final CounterWrapper numberOfNodeAdminMaintenanceFails; - private final Docker docker; + private final DockerOperations dockerOperations; private final ProcessExecuter processExecuter; private final Environment environment; private final Clock clock; @@ -58,8 +58,8 @@ public class StorageMaintainer { private Map<ContainerName, MaintenanceThrottler> maintenanceThrottlerByContainerName = new ConcurrentHashMap<>(); - public StorageMaintainer(Docker docker, ProcessExecuter processExecuter, MetricReceiverWrapper metricReceiver, Environment environment, Clock clock) { - this.docker = docker; + public StorageMaintainer(DockerOperations dockerOperations, ProcessExecuter processExecuter, MetricReceiverWrapper metricReceiver, Environment environment, Clock clock) { + this.dockerOperations = dockerOperations; this.processExecuter = processExecuter; this.environment = environment; this.clock = clock; @@ -99,7 +99,7 @@ public class StorageMaintainer { vespaSchedule.writeTo(yamasAgentFolder); hostLifeSchedule.writeTo(yamasAgentFolder); final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"}; - docker.executeInContainerAsRoot(containerName, restartYamasAgent); + dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent); } catch (IOException e) { throw new RuntimeException("Failed to write secret-agent schedules for " + containerName, e); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java new file mode 100644 index 00000000000..9caf1307aa4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminConfig.java @@ -0,0 +1,37 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeadmin; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.util.logging.Logger; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class NodeAdminConfig { + private static final Logger logger = Logger.getLogger(NodeAdminConfig.class.getName()); + private static final ObjectMapper mapper = new ObjectMapper(); + + enum Mode { + tenant, + config_server_host + } + + @JsonProperty("mode") + public Mode mode = Mode.tenant; + + public static NodeAdminConfig fromFile(File file) { + if (!file.exists()) { + return new NodeAdminConfig(); + } + + try { + return mapper.readValue(file, NodeAdminConfig.class); + } catch (IOException e) { + throw new RuntimeException("Failed to read " + file + " as a " + + NodeAdminConfig.class.getName(), e); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java new file mode 100644 index 00000000000..1ce6c9a14e1 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminMain.java @@ -0,0 +1,118 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.nodeadmin; + +import com.yahoo.concurrent.classlock.ClassLocking; +import com.yahoo.net.HostName; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.io.File; +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; +import java.util.function.Function; + +/** + * NodeAdminMain is the main component of the node admin JDisc application: + * - It will read config and check its environment to figure out its responsibilities + * - It will "start" (only) the necessary components. + * - Other components MUST NOT try to start (typically in constructor) since the features + * they provide is NOT WANTED and possibly destructive, and/or the environment may be + * incompatible. For instance, trying to contact the Docker daemon too early will + * be fatal: the node admin may not have installed and started the docker daemon. + */ +public class NodeAdminMain implements AutoCloseable { + private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); + private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); + + private final Docker docker; + private final MetricReceiverWrapper metricReceiver; + private final ClassLocking classLocking; + + private Optional<NodeAdminStateUpdater> nodeAdminStateUpdater = Optional.empty(); + + public NodeAdminMain(Docker docker, MetricReceiverWrapper metricReceiver, ClassLocking classLocking) { + this.docker = docker; + this.metricReceiver = metricReceiver; + this.classLocking = classLocking; + } + + @Override + public void close() { + nodeAdminStateUpdater.ifPresent(NodeAdminStateUpdater::stop); + } + + public NodeAdminStateUpdater getNodeAdminStateUpdater() { + return nodeAdminStateUpdater.get(); + } + + public void start() { + String staticConfigPath = Defaults.getDefaults().underVespaHome("conf/node-admin.json"); + NodeAdminConfig config = NodeAdminConfig.fromFile(new File(staticConfigPath)); + + switch (config.mode) { + case tenant: + setupTenantHostNodeAdmin(); + break; + case config_server_host: + setupConfigServerHostNodeAdmin(); + break; + default: + throw new IllegalStateException( + "Unknown bootstrap mode: " + config.mode.name()); + } + } + + private void setupTenantHostNodeAdmin() { + nodeAdminStateUpdater = Optional.of(createNodeAdminStateUpdater()); + nodeAdminStateUpdater.get().start(); + } + + private NodeAdminStateUpdater createNodeAdminStateUpdater() { + Clock clock = Clock.systemUTC(); + String dockerHostHostName = HostName.getLocalhost(); + ProcessExecuter processExecuter = new ProcessExecuter(); + Environment environment = new Environment(); + + ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris()); + NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); + Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); + + docker.start(); + DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter); + + StorageMaintainer storageMaintainer = new StorageMaintainer(dockerOperations, processExecuter, metricReceiver, environment, clock); + AclMaintainer aclMaintainer = new AclMaintainer(dockerOperations, nodeRepository, dockerHostHostName); + + Function<String, NodeAgent> nodeAgentFactory = + (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, + storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); + NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, + metricReceiver, clock); + + return new NodeAdminStateUpdater(nodeRepository, orchestrator, storageMaintainer, nodeAdmin, + dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, classLocking); + } + + private void setupConfigServerHostNodeAdmin() { + // TODO: + // - install and start docker daemon + // - Read config that specifies which containers to start how + // - use thin static backends for node repo, orchestrator, and others + // - Start core node admin. + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java index 3777d7e20d1..ea9b38efa26 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java @@ -4,78 +4,29 @@ package com.yahoo.vespa.hosted.node.admin.provider; import com.google.inject.Inject; import com.yahoo.concurrent.classlock.ClassLocking; import com.yahoo.container.di.componentgraph.Provider; -import com.yahoo.net.HostName; - -import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminMain; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.Environment; - -import java.time.Clock; -import java.time.Duration; -import java.util.function.Function; -/** - * Set up node admin for production. - * - * @author dybis - */ public class NodeAdminProvider implements Provider<NodeAdminStateUpdater> { - - // WARNING: reducing the node agent interval will increase the load on the config servers - private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(60); - private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); - - private final NodeAdminStateUpdater nodeAdminStateUpdater; + private final NodeAdminMain nodeAdminMain; @Inject - public NodeAdminProvider(Docker docker, MetricReceiverWrapper metricReceiver, ClassLocking classLocking) { - Clock clock = Clock.systemUTC(); - String dockerHostHostName = HostName.getLocalhost(); - ProcessExecuter processExecuter = new ProcessExecuter(); - Environment environment = new Environment(); - - ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerUris()); - NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor); - Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); - DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter); - - StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, metricReceiver, environment, clock); - AclMaintainer aclMaintainer = new AclMaintainer(dockerOperations, nodeRepository, dockerHostHostName); - - Function<String, NodeAgent> nodeAgentFactory = - (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, - metricReceiver, clock); - - nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, orchestrator, storageMaintainer, nodeAdmin, - dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL, classLocking); - - nodeAdminStateUpdater.start(); + public NodeAdminProvider(Docker docker, + MetricReceiverWrapper metricReceiver, + ClassLocking classLocking) { + nodeAdminMain = new NodeAdminMain(docker, metricReceiver, classLocking); + nodeAdminMain.start(); } @Override public NodeAdminStateUpdater get() { - return nodeAdminStateUpdater; + return nodeAdminMain.getNodeAdminStateUpdater(); } @Override public void deconstruct() { - nodeAdminStateUpdater.stop(); + nodeAdminMain.close(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index a1cc1850d23..7a5d713936d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -33,6 +33,9 @@ public class DockerMock implements Docker { } @Override + public void start() { } + + @Override public CreateContainerCommand createContainerCommand( DockerImage dockerImage, ContainerResources containerResources, diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index b4a5b552738..7a314ff0614 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -57,12 +57,12 @@ public class DockerTester implements AutoCloseable { .inetAddressResolver(inetAddressResolver) .pathResolver(new PathResolver(Paths.get("/tmp"), Paths.get("/tmp"))).build(); Clock clock = Clock.systemUTC(); - StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerMock, null, environment, callOrderVerifier, clock); + DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, null); + StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, null, environment, callOrderVerifier, clock); AclMaintainer aclMaintainer = mock(AclMaintainer.class); MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, null); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java index ffb4105888b..67627ee1a83 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.util.Environment; @@ -19,8 +19,8 @@ import java.util.Optional; public class StorageMaintainerMock extends StorageMaintainer { private final CallOrderVerifier callOrderVerifier; - public StorageMaintainerMock(Docker docker, ProcessExecuter processExecuter, Environment environment, CallOrderVerifier callOrderVerifier, Clock clock) { - super(docker, processExecuter, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment, clock); + public StorageMaintainerMock(DockerOperations dockerOperations, ProcessExecuter processExecuter, Environment environment, CallOrderVerifier callOrderVerifier, Clock clock) { + super(dockerOperations, processExecuter, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment, clock); this.callOrderVerifier = callOrderVerifier; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 69969336efc..38dd11a7a51 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -6,9 +6,9 @@ import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.system.ProcessExecuter; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.node.admin.util.PathResolver; import com.yahoo.vespa.hosted.provision.Node; @@ -21,7 +21,7 @@ import java.io.IOException; import java.nio.file.Files; import java.time.Duration; -import static org.junit.Assert.*; +import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -35,7 +35,7 @@ public class StorageMaintainerTest { private final ManualClock clock = new ManualClock(); private final Environment environment = new Environment.Builder() .pathResolver(new PathResolver()).build(); - private final Docker docker = mock(Docker.class); + private final DockerOperations docker = mock(DockerOperations.class); private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); private final StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, new MetricReceiverWrapper(MetricReceiver.nullImplementation), environment, clock); diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 885d3e9aad7..7f4ea9dcc2e 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -8,6 +8,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/test/make_bucket_space.h> +#include <vespa/metrics/loadmetric.h> #include <vespa/vdslib/state/state.h> #include <vespa/vdslib/state/node.h> #include <vespa/vdslib/state/nodestate.h> diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h index 75d3eac4538..ca4c79e3005 100644 --- a/persistence/src/vespa/persistence/spi/context.h +++ b/persistence/src/vespa/persistence/spi/context.h @@ -29,7 +29,6 @@ #pragma once -#include <vespa/metrics/loadmetric.h> #include <persistence/spi/types.h> #include <vespa/persistence/spi/read_consistency.h> #include <vespa/vespalib/trace/trace.h> @@ -38,8 +37,7 @@ namespace metrics { class LoadType; } -namespace storage { -namespace spi { +namespace storage::spi { using LoadType = metrics::LoadType; @@ -93,6 +91,4 @@ public: { _trace.trace(level, msg, addTime); } }; -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp index 76b0a3c4686..58e662a2b1d 100644 --- a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "metricpersistenceprovider.h" +#include <vespa/metrics/valuemetric.h> +#include <vespa/metrics/metrictimer.h> #include <cassert> #include <vespa/log/log.h> diff --git a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h index e169ad098c7..b804fd21550 100644 --- a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h @@ -6,10 +6,10 @@ #pragma once #include "persistenceprovider.h" -#include <vespa/metrics/metrics.h> +#include <vespa/metrics/metricset.h> +#include <vespa/metrics/valuemetric.h> -namespace storage { -namespace spi { +namespace storage::spi { class MetricPersistenceProvider : public PersistenceProvider, public metrics::MetricSet @@ -61,5 +61,5 @@ private: void defineResultMetrics(int index, const char* name); }; -} // spi -} // storage +} + diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index bd274093f87..4b73e4ca115 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -13,6 +13,7 @@ #include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/metrics/loadmetric.h> #include <vespa/vespalib/testkit/testapp.h> #include <algorithm> #include <set> diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index c7b01d209ee..e2b389fb898 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -3,8 +3,8 @@ #include "persistenceengine.h" #include "ipersistenceengineowner.h" #include "transport_latch.h" +#include <vespa/metrics/loadmetric.h> #include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP(".proton.persistenceengine.persistenceengine"); @@ -23,6 +23,8 @@ using vespalib::IllegalStateException; using vespalib::Sequence; using vespalib::make_string; +using namespace std::chrono_literals; + namespace proton { namespace { @@ -623,7 +625,7 @@ PersistenceEngine::destroyIterators() Result res(destroyIterator(id, context)); if (res.hasError()) { LOG(debug, "%ld iterator left. Can not destroy iterator '%ld'. Reason='%s'", _iterators.size(), id.getValue(), res.toString().c_str()); - FastOS_Thread::Sleep(100); // Sleep 0.1 seconds + std::this_thread::sleep_for(100ms); } } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/ImportResult.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/ImportResult.java index b4a9b363ade..947e6d7a5e1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/ImportResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/ImportResult.java @@ -12,40 +12,91 @@ import java.util.Map; import java.util.stream.Collectors; /** - * The result of importing a TensorFlow model into Vespa: - * - A list of ranking expressions reproducing the computations of the outputs in the TensorFlow model - * - A list of named constant tensors - * - A list of expected input tensors, with their tensor type - * - A list of warning messages + * The result of importing a TensorFlow model into Vespa. + * - A set of signatures which are named collections of inputs and outputs. + * - A set of named constant tensors represented by Variable nodes in TensorFlow. + * - A list of warning messages. * * @author bratseth */ // This object can be built incrementally within this package, but is immutable when observed from outside the package -// TODO: Retain signature structure in ImportResult (input + output-expression bundles) public class ImportResult { - private final List<RankingExpression> expressions = new ArrayList<>(); - private final Map<String, Tensor> constants = new HashMap<>(); + private final Map<String, Signature> signatures = new HashMap<>(); private final Map<String, TensorType> arguments = new HashMap<>(); + private final Map<String, Tensor> constants = new HashMap<>(); + private final Map<String, RankingExpression> expressions = new HashMap<>(); private final List<String> warnings = new ArrayList<>(); - void add(RankingExpression expression) { expressions.add(expression); } - void set(String name, Tensor constant) { constants.put(name, constant); } - void set(String name, TensorType argument) { arguments.put(name, argument); } + void argument(String name, TensorType argumentType) { arguments.put(name, argumentType); } + void constant(String name, Tensor constant) { constants.put(name, constant); } + void expression(String name, RankingExpression expression) { expressions.put(name, expression); } void warn(String warning) { warnings.add(warning); } - /** Returns an immutable list of the expressions of this */ - public List<RankingExpression> expressions() { return Collections.unmodifiableList(expressions); } + /** Returns the given signature. If it does not already exist it is added to this. */ + Signature signature(String name) { + return signatures.computeIfAbsent(name, n -> new Signature(n)); + } + + /** Returns an immutable map of the arguments ("Placeholders") of this */ + public Map<String, TensorType> arguments() { return Collections.unmodifiableMap(arguments); } /** Returns an immutable map of the constants of this */ public Map<String, Tensor> constants() { return Collections.unmodifiableMap(constants); } - /** Returns an immutable map of the arguments of this */ - public Map<String, TensorType> arguments() { return Collections.unmodifiableMap(arguments); } + /** + * Returns an immutable map of the expressions of this - corresponding to TensorFlow nodes + * which are not Placeholders or Variables (which instead become respectively arguments and constants). + * Note that only nodes recursively referenced by a placeholder are added. + */ + public Map<String, RankingExpression> expressions() { return Collections.unmodifiableMap(expressions); } /** Returns an immutable list, in natural sort order of the warnings generated while importing this */ public List<String> warnings() { return warnings.stream().sorted().collect(Collectors.toList()); } + /** Returns an immutable map of the signatures of this */ + public Map<String, Signature> signatures() { return Collections.unmodifiableMap(signatures); } + + /** + * A signature is a set of named inputs and outputs, where the inputs maps to argument ("placeholder") names+types, + * and outputs maps to expressions nodes. + */ + public class Signature { + + private final String name; + private final Map<String, String> inputs = new HashMap<>(); + private final Map<String, String> outputs = new HashMap<>(); + + Signature(String name) { + this.name = name; + } + + void input(String inputName, String argumentName) { inputs.put(inputName, argumentName); } + void output(String name, String expressionName) { outputs.put(name, expressionName); } + + /** Returns the result this is part of */ + ImportResult owner() { return ImportResult.this; } + + /** + * Returns an immutable map of the inputs (evaluation context) of this. This is a map from input name + * to argument (Placeholder) name in the owner of this + */ + public Map<String, String> inputs() { return Collections.unmodifiableMap(inputs); } + + /** Returns owner().arguments().get(inputs.get(name)), e.g the type of the argument this input references */ + public TensorType inputArgument(String inputName) { return owner().arguments().get(inputs.get(inputName)); } + + /** Returns an immutable list of the expression names of this */ + public Map<String, String> outputs() { return Collections.unmodifiableMap(outputs); } + + /** Returns owner().expressions().get(outputs.get(outputName)), e.g the expression this output references */ + public RankingExpression outputExpression(String outputName) { return owner().expressions().get(outputs.get(outputName)); } + + @Override + public String toString() { return "signature '" + name + "'"; } + + } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java index e7f7b5ef2f4..bac141644c6 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java @@ -90,8 +90,8 @@ class OperationMapper { String name = tfNode.getName(); TensorType type = result.arguments().get(name); if (type == null) - throw new IllegalArgumentException("An placeholder operation node is referencing input '" + name + - "', but there is no such input"); + throw new IllegalArgumentException("A 'placeholder' node is referencing placeholder '" + name + + "', but there is no such placeholder"); // Included literally in the expression and so must be produced by a separate macro in the rank profile return new TypedTensorFunction(type, new VariableTensor(name)); } @@ -114,7 +114,7 @@ class OperationMapper { throw new IllegalStateException("Expected 1 tensor from reading Variable " + name + ", but got " + importedTensors.size()); Tensor constant = tensorConverter.toVespaTensor(importedTensors.get(0)); - result.set(name, constant); + result.constant(name, constant); return new TypedTensorFunction(constant.type(), new TensorFunctionNode.TensorFunctionExpressionNode(new ReferenceNode("constant(" + name + ")"))); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorConverter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorConverter.java index df43225c333..1960cf94591 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorConverter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorConverter.java @@ -26,7 +26,7 @@ public class TensorConverter { int dimensionIndex = 0; for (long dimensionSize : shape) { if (dimensionSize == 0) dimensionSize = 1; // TensorFlow ... - b.indexed("d" + (dimensionIndex++), (int) dimensionSize); + b.indexed("d" + (dimensionIndex++), dimensionSize); } return b.build(); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java index 33523244129..4a6551adca7 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowImporter.java @@ -1,11 +1,9 @@ package com.yahoo.searchlib.rankingexpression.integration.tensorflow; import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.ScalarFunctions; -import com.yahoo.tensor.functions.TensorFunction; import com.yahoo.yolean.Exceptions; import org.tensorflow.SavedModelBundle; import org.tensorflow.framework.GraphDef; @@ -38,49 +36,53 @@ public class TensorFlowImporter { */ public ImportResult importModel(String modelDir) { try (SavedModelBundle model = SavedModelBundle.load(modelDir, "serve")) { - return importGraph(MetaGraphDef.parseFrom(model.metaGraphDef()), model); + return importModel(model); } - catch (IOException e) { - throw new IllegalArgumentException("Could not read TensorFlow model from directory '" + modelDir + "'", e); + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Could not import TensorFlow model from directory '" + modelDir + "'", e); } } - public ImportResult importNode(String modelDir, String inputSignatureName, String nodeName) { - try (SavedModelBundle model = SavedModelBundle.load(modelDir, "serve")) { - MetaGraphDef graph = MetaGraphDef.parseFrom(model.metaGraphDef()); - SignatureDef signature = graph.getSignatureDefMap().get(inputSignatureName); - ImportResult result = new ImportResult(); - importInputs(signature.getInputsMap(), result); - result.add(new RankingExpression(nodeName, importNode(nodeName, graph.getGraphDef(), model, result))); - return result; + /** Imports a TensorFlow model */ + public ImportResult importModel(SavedModelBundle model) { + try { + return importGraph(MetaGraphDef.parseFrom(model.metaGraphDef()), model); } catch (IOException e) { - throw new IllegalArgumentException("Could not read TensorFlow model from directory '" + modelDir + "'", e); + throw new IllegalArgumentException("Could not import TensorFlow model '" + model + "'", e); } } private ImportResult importGraph(MetaGraphDef graph, SavedModelBundle model) { ImportResult result = new ImportResult(); for (Map.Entry<String, SignatureDef> signatureEntry : graph.getSignatureDefMap().entrySet()) { - importInputs(signatureEntry.getValue().getInputsMap(), result); + ImportResult.Signature signature = result.signature(signatureEntry.getKey()); // Prefer key over "methodName" + + importInputs(signatureEntry.getValue().getInputsMap(), signature); for (Map.Entry<String, TensorInfo> output : signatureEntry.getValue().getOutputsMap().entrySet()) { + String outputName = output.getKey(); try { - ExpressionNode node = importOutput(output.getValue(), graph.getGraphDef(), model, result); - result.add(new RankingExpression(output.getKey(), node)); + NodeDef node = getNode(nameOf(output.getValue().getName()), graph.getGraphDef()); + importNode(node, graph.getGraphDef(), model, result); + signature.output(outputName, nameOf(output.getValue().getName())); } catch (IllegalArgumentException e) { - result.warn("Skipping output '" + output.getValue().getName() + "' of signature '" + - signatureEntry.getValue().getMethodName() + - "': " + Exceptions.toMessageString(e)); + result.warn("Skipping output '" + outputName + "' of " + signature + + ": " + Exceptions.toMessageString(e)); } } } return result; } - private void importInputs(Map<String, TensorInfo> inputInfoMap, ImportResult result) { - inputInfoMap.forEach((key, value) -> result.set(nameOf(value.getName()), - importTensorType(value.getTensorShape()))); + private void importInputs(Map<String, TensorInfo> inputInfoMap, ImportResult.Signature signature) { + inputInfoMap.forEach((key, value) -> { + String argumentName = nameOf(value.getName()); + TensorType argumentType = importTensorType(value.getTensorShape()); + // Arguments are (Placeholder) nodes, so not local to the signature: + signature.owner().argument(argumentName, argumentType); + signature.input(key, argumentName); + }); } private TensorType importTensorType(TensorShapeProto tensorShape) { @@ -95,18 +97,13 @@ public class TensorFlowImporter { return b.build(); } - private ExpressionNode importOutput(TensorInfo output, GraphDef graph, SavedModelBundle model, ImportResult result) { - return importNode(nameOf(output.getName()), graph, model, result); - } - - private ExpressionNode importNode(String nodeName, GraphDef graph, SavedModelBundle model, ImportResult result) { - TensorFunction function = importNode(getNode(nodeName, graph), graph, model, result).function(); - return new TensorFunctionNode(function); // wrap top level (only) as an expression - } - /** Recursively convert a graph of TensorFlow nodes into a Vespa tensor function expression tree */ private TypedTensorFunction importNode(NodeDef tfNode, GraphDef graph, SavedModelBundle model, ImportResult result) { - return tensorFunctionOf(tfNode, graph, model, result); + TypedTensorFunction function = tensorFunctionOf(tfNode, graph, model, result); + // We add all intermediate nodes imported as separate expressions. Only those referenced in a signature output + // will be used + result.expression(tfNode.getName(), new RankingExpression(tfNode.getName(), new TensorFunctionNode(function.function()))); + return function; } private TypedTensorFunction tensorFunctionOf(NodeDef tfNode, GraphDef graph, SavedModelBundle model, ImportResult result) { @@ -123,7 +120,8 @@ public class TensorFlowImporter { } } - private List<TypedTensorFunction> importArguments(NodeDef tfNode, GraphDef graph, SavedModelBundle model, ImportResult result) { + private List<TypedTensorFunction> importArguments(NodeDef tfNode, GraphDef graph, SavedModelBundle model, + ImportResult result) { return tfNode.getInputList().stream() .map(argNode -> importNode(getNode(nameOf(argNode), graph), graph, model, result)) .collect(Collectors.toList()); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java index d366c9bfbe5..9da1ba40144 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.rule; -import com.google.common.collect.ImmutableList; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; import com.yahoo.searchlib.rankingexpression.evaluation.Value; @@ -10,7 +9,6 @@ import com.yahoo.tensor.TensorType; import java.util.Collections; import java.util.Deque; import java.util.List; -import java.util.function.*; /** * A tensor generating function, whose arguments are determined by a tensor type @@ -57,14 +55,14 @@ public class GeneratorLambdaFunctionNode extends CompositeNode { /** * Returns this as an operator which converts a list of integers into a double */ - public IntegerListToDoubleLambda asIntegerListToDoubleOperator() { - return new IntegerListToDoubleLambda(); + public LongListToDoubleLambda asLongListToDoubleOperator() { + return new LongListToDoubleLambda(); } - private class IntegerListToDoubleLambda implements java.util.function.Function<List<Integer>, Double> { + private class LongListToDoubleLambda implements java.util.function.Function<List<Long>, Double> { @Override - public Double apply(List<Integer> arguments) { + public Double apply(List<Long> arguments) { MapContext context = new MapContext(); for (int i = 0; i < type.dimensions().size(); i++) context.put(type.dimensions().get(i).name(), arguments.get(i)); diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index 7821ab88b86..541738db8e0 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -467,7 +467,7 @@ ExpressionNode tensorGenerate() : } { <TENSOR> type = tensorTypeArgument() <LBRACE> generator = expression() <RBRACE> - { return new TensorFunctionNode(new Generate(type, new GeneratorLambdaFunctionNode(type, generator).asIntegerListToDoubleOperator())); } + { return new TensorFunctionNode(new Generate(type, new GeneratorLambdaFunctionNode(type, generator).asLongListToDoubleOperator())); } } ExpressionNode tensorRange() : diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/Mnist_SoftmaxTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/Mnist_SoftmaxTestCase.java index d50a97cc8e0..0370fc7fc94 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/Mnist_SoftmaxTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/Mnist_SoftmaxTestCase.java @@ -27,18 +27,13 @@ public class Mnist_SoftmaxTestCase { @Test public void testImporting() { String modelDir = "src/test/files/integration/tensorflow/mnist_softmax/saved"; - ImportResult result = new TensorFlowImporter().importModel(modelDir); + SavedModelBundle model = SavedModelBundle.load(modelDir, "serve"); + ImportResult result = new TensorFlowImporter().importModel(model); // Check logged messages result.warnings().forEach(System.err::println); assertEquals(0, result.warnings().size()); - // Check arguments - assertEquals(1, result.arguments().size()); - TensorType argument0 = result.arguments().get("Placeholder"); - assertNotNull(argument0); - assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), argument0); - // Check constants assertEquals(2, result.constants().size()); @@ -54,38 +49,45 @@ public class Mnist_SoftmaxTestCase { constant1.type()); assertEquals(10, constant1.size()); - // Check resulting Vespa expression - assertEquals(1, result.expressions().size()); - assertEquals("y", result.expressions().get(0).getName()); + // Check signatures + assertEquals(1, result.signatures().size()); + ImportResult.Signature signature = result.signatures().get("serving_default"); + assertNotNull(signature); + + // ... signature inputs + assertEquals(1, signature.inputs().size()); + TensorType argument0 = signature.inputArgument("x"); + assertNotNull(argument0); + assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), argument0); + + // ... signature outputs + assertEquals(1, signature.outputs().size()); + RankingExpression output = signature.outputExpression("y"); + assertNotNull(output); + assertEquals("add", output.getName()); assertEquals("" + "join(rename(matmul(Placeholder, rename(constant(Variable), (d0, d1), (d1, d3)), d1), d3, d1), " + "rename(constant(Variable_1), d0, d1), " + "f(a,b)(a + b))", - toNonPrimitiveString(result.expressions().get(0))); + toNonPrimitiveString(output)); // Test execution - String signatureName = "serving_default"; - - assertEqualResult(modelDir, signatureName, "Variable/read"); - assertEqualResult(modelDir, signatureName, "Variable_1/read"); - // TODO: Assert that argument fed is as expected assertEqualResult(modelDir, signatureName, "Placeholder"); - assertEqualResult(modelDir, signatureName, "MatMul"); - assertEqualResult(modelDir, signatureName, "add"); + assertEqualResult(model, result, "Variable/read"); + assertEqualResult(model, result, "Variable_1/read"); + assertEqualResult(model, result, "MatMul"); + assertEqualResult(model, result, "add"); } - private void assertEqualResult(String modelDir, String signatureName, String operationName) { - ImportResult result = new TensorFlowImporter().importNode(modelDir, signatureName, operationName); - - Tensor tfResult = tensorFlowExecute(modelDir, operationName); + private void assertEqualResult(SavedModelBundle model, ImportResult result, String operationName) { + Tensor tfResult = tensorFlowExecute(model, operationName); Context context = contextFrom(result); Tensor placeholder = placeholderArgument(); context.put("Placeholder", new TensorValue(placeholder)); - Tensor vespaResult = result.expressions().get(0).evaluate(context).asTensor(); + Tensor vespaResult = result.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", vespaResult, tfResult); } - private Tensor tensorFlowExecute(String modelDir, String operationName) { - SavedModelBundle model = SavedModelBundle.load(modelDir, "serve"); + private Tensor tensorFlowExecute(SavedModelBundle model, String operationName) { Session.Runner runner = model.session().runner(); org.tensorflow.Tensor<?> placeholder = org.tensorflow.Tensor.create(new long[]{ 1, 784 }, FloatBuffer.allocate(784)); runner.feed("Placeholder", placeholder); diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp index 9957d162d9d..702ff0fc5cf 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp @@ -6,6 +6,7 @@ #include "postinglistattribute.h" #include <vespa/searchlib/common/growablebitvector.h> #include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp index 990117a71ce..1ab3c6b8b51 100644 --- a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp @@ -2,6 +2,7 @@ #include "foregroundtaskexecutor.h" #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::ThreadStackExecutor; diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp index 45004db2615..446c9ec39ec 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "sequencedtaskexecutor.h" +#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::BlockingThreadStackExecutor; diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 8f73c9862ae..7e881d8de76 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -209,8 +209,7 @@ VisitCache::Cache::locateAndInvalidateOtherSubsets(const LockGuard & cacheGuard, CompressedBlobSet VisitCache::read(const IDocumentStore::LidVector & lids) const { - KeySet key(lids); - return _cache->readSet(lids); + return _cache->readSet(KeySet(lids)); } void diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index 1bf867c5580..effc6c19a21 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -20,7 +20,7 @@ class KeySet { public: KeySet() : _keys() { } KeySet(uint32_t key); - KeySet(const IDocumentStore::LidVector &keys); + explicit KeySet(const IDocumentStore::LidVector &keys); uint32_t hash() const { return _keys.empty() ? 0 : _keys[0]; } bool operator==(const KeySet &rhs) const { return _keys == rhs._keys; } bool operator<(const KeySet &rhs) const { return _keys < rhs._keys; } diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp index fe57de093dd..61147229497 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp +++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp @@ -110,6 +110,7 @@ void lrucache_map<P>::erase(const K & key) { internal_iterator it = HashTable::find(key); if (it != HashTable::end()) { + next_t h = HashTable::hash(key); onRemove(key); LV & v = it->second; if (v._prev != LinkedValueBase::npos) { @@ -122,7 +123,7 @@ lrucache_map<P>::erase(const K & key) { } else { _tail = v._prev; } - HashTable::erase(*this, it); + HashTable::erase(*this, h, it); } } @@ -202,7 +203,7 @@ lrucache_map<P>::removeOld() { { _tail = last->second._prev; HashTable::getByInternalIndex(_tail).second._next = LinkedValueBase::npos; - HashTable::erase(*this, HashTable::find(last->first)); + HashTable::erase(*this, HashTable::hash(last->first), HashTable::find(last->first)); } } } diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala index e1d5ba6577d..1d4b83ce7d3 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala @@ -158,7 +158,7 @@ object StandaloneContainerApplication { val logger = new BaseDeployLogger val rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile).includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build() // TODO: Needed until we get rid of semantic rules - val applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone().id, new RuleConfigDeriver { + val applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), new RuleConfigDeriver { override def derive(ruleBaseDir: String, outputDir: String): Unit = {} }, logger) validateApplication(applicationPackage) diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index b9e33ea8d26..ff442114c4c 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -4,6 +4,7 @@ #include <iomanip> #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/distributor/bucketdbupdater.h> +#include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/pending_bucket_space_db_transition.h> #include <vespa/storage/distributor/outdated_nodes_map.h> #include <vespa/vespalib/io/fileutil.h> @@ -22,8 +23,7 @@ using namespace storage::lib; using document::test::makeDocumentBucket; using document::test::makeBucketSpace; -namespace storage { -namespace distributor { +namespace storage::distributor { class BucketDBUpdaterTest : public CppUnit::TestFixture, public DistributorTestUtil @@ -2499,5 +2499,4 @@ void BucketDBUpdaterTest::batch_update_from_distributor_change_does_not_mark_div "0:5/1/2/3|1:5/7/8/9", true)); } -} // distributor -} // storage +} diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp index 683352e6b09..a0b8cd424ac 100644 --- a/storage/src/tests/distributor/externaloperationhandlertest.cpp +++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp @@ -2,15 +2,14 @@ #include <tests/distributor/distributortestutil.h> #include <vespa/storage/distributor/externaloperationhandler.h> -#include <vespa/storage/distributor/operation_sequencer.h> -#include <vespa/storageapi/message/persistence.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/storageapi/message/persistence.h> #include <vespa/document/test/make_document_bucket.h> using document::test::makeDocumentBucket; -namespace storage { -namespace distributor { +namespace storage::distributor { class ExternalOperationHandlerTest : public CppUnit::TestFixture, public DistributorTestUtil @@ -471,5 +470,4 @@ void ExternalOperationHandlerTest::sequencing_can_be_explicitly_config_disabled( // pseudo-locks in the sequencer. I.e. if we get a RemoveLocation with id.user==123456, this // prevents any handles from being acquired to any GID under location BucketId(32, 123456). -} // distributor -} // storage +} diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp index 8bb8e24c17a..80c093dea87 100644 --- a/storage/src/tests/distributor/getoperationtest.cpp +++ b/storage/src/tests/distributor/getoperationtest.cpp @@ -5,16 +5,13 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/storage/distributor/externaloperationhandler.h> #include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributormetricsset.h> #include <tests/distributor/distributortestutil.h> #include <vespa/storageapi/message/persistence.h> -#include <tests/common/dummystoragelink.h> #include <vespa/document/test/make_document_bucket.h> -#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/config/helper/configgetter.hpp> #include <iomanip> -#include <iostream> -#include <memory> #include <vespa/storage/distributor/operations/external/getoperation.h> using std::shared_ptr; @@ -23,8 +20,7 @@ using document::DocumenttypesConfig; using config::FileSpec; using document::test::makeDocumentBucket; -namespace storage { -namespace distributor { +namespace storage::distributor { class GetOperationTest : public CppUnit::TestFixture, public DistributorTestUtil { CPPUNIT_TEST_SUITE(GetOperationTest); @@ -568,5 +564,4 @@ GetOperationTest::canGetDocumentsWhenAllReplicaNodesRetired() _sender.getCommands(true)); } -} // distributor -} // storage +} diff --git a/storage/src/tests/distributor/messagesenderstub.h b/storage/src/tests/distributor/messagesenderstub.h index e5bae9c6702..b86863890a1 100644 --- a/storage/src/tests/distributor/messagesenderstub.h +++ b/storage/src/tests/distributor/messagesenderstub.h @@ -3,6 +3,7 @@ #include <vespa/storage/distributor/distributormessagesender.h> #include <cassert> +#include <vector> namespace storage { diff --git a/storage/src/tests/distributor/visitoroperationtest.cpp b/storage/src/tests/distributor/visitoroperationtest.cpp index 972ccf41bfe..17d1bc288ca 100644 --- a/storage/src/tests/distributor/visitoroperationtest.cpp +++ b/storage/src/tests/distributor/visitoroperationtest.cpp @@ -9,6 +9,7 @@ #include <vespa/storageapi/message/state.h> #include <vespa/storage/distributor/operations/external/visitoroperation.h> #include <vespa/storage/distributor/operations/external/visitororder.h> +#include <vespa/storage/distributor/distributormetricsset.h> #include <tests/distributor/distributortestutil.h> #include <vespa/storage/distributor/distributor.h> #include <tests/common/dummystoragelink.h> @@ -21,8 +22,7 @@ using namespace storage::lib; using namespace std::string_literals; using document::test::makeBucketSpace; -namespace storage { -namespace distributor { +namespace storage::distributor { class VisitorOperationTest : public CppUnit::TestFixture, public DistributorTestUtil { @@ -1674,5 +1674,4 @@ VisitorOperationTest::statistical_metrics_not_updated_on_wrong_distribution() CPPUNIT_ASSERT_EQUAL(0.0, defaultVisitorMetrics().latency.getCount()); } -} // distributor -} // storage +} diff --git a/storage/src/tests/persistence/splitbitdetectortest.cpp b/storage/src/tests/persistence/splitbitdetectortest.cpp index c20aae373ec..01baa8f4e98 100644 --- a/storage/src/tests/persistence/splitbitdetectortest.cpp +++ b/storage/src/tests/persistence/splitbitdetectortest.cpp @@ -8,6 +8,7 @@ #include <vespa/persistence/spi/test.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/metrics/loadmetric.h> #include <algorithm> using storage::spi::test::makeSpiBucket; diff --git a/storage/src/vespa/storage/common/bucketmessages.cpp b/storage/src/vespa/storage/common/bucketmessages.cpp index 3157bad49e5..e92e2d4c3bf 100644 --- a/storage/src/vespa/storage/common/bucketmessages.cpp +++ b/storage/src/vespa/storage/common/bucketmessages.cpp @@ -2,6 +2,7 @@ #include "bucketmessages.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <ostream> using document::BucketSpace; diff --git a/storage/src/vespa/storage/common/messagesender.h b/storage/src/vespa/storage/common/messagesender.h index 8c45995c42f..659fccad412 100644 --- a/storage/src/vespa/storage/common/messagesender.h +++ b/storage/src/vespa/storage/common/messagesender.h @@ -18,13 +18,14 @@ #include <memory> -namespace storage { -namespace api { +namespace storage::api { class StorageCommand; class StorageReply; class StorageMessage; } +namespace storage { + struct MessageSender { virtual ~MessageSender() {} diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index 46fa0f72d76..cc1181e0d58 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -2,9 +2,9 @@ #include "bucketdbupdater.h" #include "distributor.h" -#include "distributor_bucket_space_repo.h" #include "distributor_bucket_space.h" #include "simpleclusterinformation.h" +#include "distributormetricsset.h" #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index 29e8d3f6221..19e2e259778 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -13,9 +13,8 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storageframework/generic/clock/timer.h> +#include <vespa/storageframework/generic/status/statusreporter.h> #include <vespa/storageapi/messageapi/messagehandler.h> -#include <set> -#include <deque> #include <list> namespace storage::distributor { diff --git a/storage/src/vespa/storage/distributor/bucketgctimecalculator.h b/storage/src/vespa/storage/distributor/bucketgctimecalculator.h index e2b232a6cf5..4ff85e568c8 100644 --- a/storage/src/vespa/storage/distributor/bucketgctimecalculator.h +++ b/storage/src/vespa/storage/distributor/bucketgctimecalculator.h @@ -4,8 +4,7 @@ #include <chrono> #include <vespa/document/bucket/bucketid.h> -namespace storage { -namespace distributor { +namespace storage::distributor { /** * Semantics are basically as follows: @@ -51,6 +50,4 @@ private: std::chrono::seconds _checkInterval; }; -} // distributor -} // storage - +} diff --git a/storage/src/vespa/storage/distributor/bucketownership.h b/storage/src/vespa/storage/distributor/bucketownership.h index c7a7773686f..bfe63c9799d 100644 --- a/storage/src/vespa/storage/distributor/bucketownership.h +++ b/storage/src/vespa/storage/distributor/bucketownership.h @@ -2,9 +2,9 @@ #pragma once #include <vespa/vdslib/state/clusterstate.h> +#include <cassert> -namespace storage { -namespace distributor { +namespace storage::distributor { class BucketOwnership { @@ -14,8 +14,7 @@ class BucketOwnership BucketOwnership(const lib::ClusterState& checkedState) : _checkedState(&checkedState), _owned(false) - { - } + { } BucketOwnership() : _checkedState(nullptr), _owned(true) {} @@ -44,6 +43,4 @@ public: } }; -} // distributor -} // storage - +} diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 1edcbe75dd6..988d39e571d 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -5,21 +5,17 @@ #include "throttlingoperationstarter.h" #include "idealstatemetricsset.h" #include "ownership_transfer_safe_time_point_calculator.h" -#include "distributor_bucket_space_repo.h" #include "distributor_bucket_space.h" -#include <vespa/storage/bucketdb/mapbucketdatabase.h> -#include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> +#include "distributormetricsset.h" #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storageframework/generic/status/xmlstatusreporter.h> - #include <vespa/log/log.h> LOG_SETUP(".distributor-main"); -namespace storage { -namespace distributor { +namespace storage::distributor { class Distributor::Status { const DelegatedStatusRequest& _request; @@ -68,34 +64,25 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _compReg(compReg), _component(compReg, "distributor"), _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>()), - _metrics(new DistributorMetricSet( - _component.getLoadTypes()->getMetricLoadTypes())), + _metrics(new DistributorMetricSet(_component.getLoadTypes()->getMetricLoadTypes())), _operationOwner(*this, _component.getClock()), _maintenanceOperationOwner(*this, _component.getClock()), _pendingMessageTracker(compReg), _bucketDBUpdater(*this, *_bucketSpaceRepo, *this, compReg), _distributorStatusDelegate(compReg, *this, *this), _bucketDBStatusDelegate(compReg, *this, _bucketDBUpdater), - _idealStateManager(*this, *_bucketSpaceRepo, compReg, - manageActiveBucketCopies), - _externalOperationHandler(*this, *_bucketSpaceRepo, - _idealStateManager, compReg), + _idealStateManager(*this, *_bucketSpaceRepo, compReg, manageActiveBucketCopies), + _externalOperationHandler(*this, *_bucketSpaceRepo, _idealStateManager, compReg), _threadPool(threadPool), _initializingIsUp(true), _doneInitializeHandler(doneInitHandler), _doneInitializing(false), _messageSender(messageSender), _bucketPriorityDb(new SimpleBucketPriorityDatabase()), - _scanner(new SimpleMaintenanceScanner( - *_bucketPriorityDb, _idealStateManager, - *_bucketSpaceRepo)), - _throttlingStarter(new ThrottlingOperationStarter( - _maintenanceOperationOwner)), - _blockingStarter(new BlockingOperationStarter(_pendingMessageTracker, - *_throttlingStarter)), - _scheduler(new MaintenanceScheduler(_idealStateManager, - *_bucketPriorityDb, - *_blockingStarter)), + _scanner(new SimpleMaintenanceScanner(*_bucketPriorityDb, _idealStateManager, *_bucketSpaceRepo)), + _throttlingStarter(new ThrottlingOperationStarter(_maintenanceOperationOwner)), + _blockingStarter(new BlockingOperationStarter(_pendingMessageTracker, *_throttlingStarter)), + _scheduler(new MaintenanceScheduler(_idealStateManager, *_bucketPriorityDb, *_blockingStarter)), _schedulingMode(MaintenanceScheduler::NORMAL_SCHEDULING_MODE), _recoveryTimeStarted(_component.getClock()), _tickResult(framework::ThreadWaitInfo::NO_MORE_CRITICAL_WORK_KNOWN), @@ -105,8 +92,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _metricLock(), _maintenanceStats(), _bucketDbStats(), - _hostInfoReporter(_pendingMessageTracker.getLatencyStatisticsProvider(), - *this), + _hostInfoReporter(_pendingMessageTracker.getLatencyStatisticsProvider(), *this), _ownershipSafeTimeCalc( std::make_unique<OwnershipTransferSafeTimePointCalculator>( std::chrono::seconds(0))) // Set by config later @@ -162,10 +148,8 @@ void Distributor::sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) { if (cmd->getType() == api::MessageType::MERGEBUCKET) { - api::MergeBucketCommand& merge( - static_cast<api::MergeBucketCommand&>(*cmd)); - _idealStateManager.getMetrics().nodesPerMerge.addValue( - merge.getNodes().size()); + api::MergeBucketCommand& merge(static_cast<api::MergeBucketCommand&>(*cmd)); + _idealStateManager.getMetrics().nodesPerMerge.addValue(merge.getNodes().size()); } sendUp(cmd); } @@ -179,10 +163,8 @@ Distributor::sendReply(const std::shared_ptr<api::StorageReply>& reply) void Distributor::setNodeStateUp() { - NodeStateUpdater::Lock::SP lock( - _component.getStateUpdater().grabStateChangeLock()); - lib::NodeState ns( - *_component.getStateUpdater().getReportedNodeState()); + NodeStateUpdater::Lock::SP lock(_component.getStateUpdater().grabStateChangeLock()); + lib::NodeState ns(*_component.getStateUpdater().getReportedNodeState()); ns.setState(lib::State::UP); _component.getStateUpdater().setReportedNodeState(ns); } @@ -832,5 +814,4 @@ Distributor::handleStatusRequest(const DelegatedStatusRequest& request) const return true; } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h index bf27dc432b6..3445397c17d 100644 --- a/storage/src/vespa/storage/distributor/distributorinterface.h +++ b/storage/src/vespa/storage/distributor/distributorinterface.h @@ -1,32 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storage/common/distributorcomponent.h> -#include <vespa/storage/common/messagesender.h> -#include <vespa/storage/distributor/pendingmessagetracker.h> -#include <vespa/storageapi/message/state.h> +#include "bucketgctimecalculator.h" +#include "distributormessagesender.h" +#include "bucketownership.h" #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <vespa/storage/distributor/bucketgctimecalculator.h> -#include <vespa/storage/distributor/distributormetricsset.h> -#include <vespa/storage/config/distributorconfiguration.h> -#include <vespa/storage/distributor/distributormessagesender.h> -#include <vespa/storage/distributor/bucketownership.h> +#include <vespa/document/bucket/bucket.h> +namespace storage::api { class MergeBucketReply; } namespace storage { + class DistributorConfiguration; + class DistributorMetricSet; +} +namespace storage::distributor { -namespace distributor { +class PendingMessageTracker; class DistributorInterface : public DistributorMessageSender { public: virtual PendingMessageTracker& getPendingMessageTracker() = 0; - virtual DistributorMetricSet& getMetrics() = 0; - virtual void enableClusterState(const lib::ClusterState& state) = 0; - virtual BucketOwnership checkOwnershipInPendingState(const document::Bucket &bucket) const = 0; - virtual void notifyDistributionChangeEnabled() = 0; /** @@ -55,19 +51,11 @@ public: * Returns true if the node is currently initializing. */ virtual bool initializing() const = 0; - virtual void handleCompletedMerge(const std::shared_ptr<api::MergeBucketReply>&) = 0; - virtual const char* getStorageNodeUpStates() const = 0; - virtual const DistributorConfiguration& getConfig() const = 0; - virtual ChainedMessageSender& getMessageSender() = 0; - virtual const BucketGcTimeCalculator::BucketIdHasher& getBucketIdHasher() const = 0; }; } - -} - diff --git a/storage/src/vespa/storage/distributor/distributormessagesender.h b/storage/src/vespa/storage/distributor/distributormessagesender.h index 0fccaad87e3..078762dd05c 100644 --- a/storage/src/vespa/storage/distributor/distributormessagesender.h +++ b/storage/src/vespa/storage/distributor/distributormessagesender.h @@ -2,11 +2,9 @@ #pragma once #include <vespa/storage/common/messagesender.h> -#include <vespa/vdslib/distribution/distribution.h> -namespace storage { - -namespace distributor { +namespace storage::lib { class NodeType; } +namespace storage::distributor { class PendingMessageTracker; @@ -16,21 +14,12 @@ public: Sends the storage command to the given node, returns message id. */ - virtual uint64_t sendToNode(const lib::NodeType& nodeType, - uint16_t node, - const std::shared_ptr<api::StorageCommand>& cmd, - bool useDocumentAPI = false); + virtual uint64_t sendToNode(const lib::NodeType& nodeType, uint16_t node, + const std::shared_ptr<api::StorageCommand>& cmd, bool useDocumentAPI = false); virtual int getDistributorIndex() const = 0; - virtual const std::string& getClusterName() const = 0; - virtual const PendingMessageTracker& getPendingMessageTracker() const = 0; }; -} // distributor - -} // storage - - - +} diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index b9607b35d28..028c9cbb0b6 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -1,18 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <deque> -#include <map> -#include <set> -#include <vespa/storage/distributor/distributorcomponent.h> -#include <vespa/storage/distributor/statechecker.h> +#include "distributorcomponent.h" +#include "statechecker.h" #include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> #include <vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vector> -namespace storage { -namespace distributor { +namespace storage::distributor { class IdealStateMetricSet; class IdealStateOperation; @@ -116,8 +112,7 @@ private: DistributorComponent _distributorComponent; DistributorBucketSpaceRepo &_bucketSpaceRepo; - std::vector<IdealStateOperation::SP> generateOperationsForBucket( - StateChecker::Context& c) const; + std::vector<IdealStateOperation::SP> generateOperationsForBucket(StateChecker::Context& c) const; bool iAmUp() const; @@ -125,9 +120,9 @@ private: // Stats tracker to use for all generateAll() calls to avoid having // to create a new hash map for each single bucket processed. NodeMaintenanceStatsTracker _statsTracker; - const IdealStateManager& _ism; - document::BucketSpace _bucketSpace; - std::ostream& _out; + const IdealStateManager & _ism; + document::BucketSpace _bucketSpace; + std::ostream & _out; public: StatusBucketVisitor(const IdealStateManager& ism, document::BucketSpace bucketSpace, std::ostream& out) : _statsTracker(), _ism(ism), _bucketSpace(bucketSpace), _out(out) {} @@ -139,11 +134,8 @@ private: }; friend class StatusBucketVisitor; - void getBucketStatus(document::BucketSpace bucketSpace, - const BucketDatabase::Entry& entry, - NodeMaintenanceStatsTracker& statsTracker, - std::ostream& out) const; + void getBucketStatus(document::BucketSpace bucketSpace, const BucketDatabase::Entry& entry, + NodeMaintenanceStatsTracker& statsTracker, std::ostream& out) const; }; -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/messagetracker.cpp b/storage/src/vespa/storage/distributor/messagetracker.cpp index b844987e978..6568cec9a80 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.cpp +++ b/storage/src/vespa/storage/distributor/messagetracker.cpp @@ -1,19 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "messagetracker.h" +#include <vespa/storageapi/messageapi/bucketcommand.h> +#include <vespa/storageapi/messageapi/bucketreply.h> #include <vespa/log/log.h> LOG_SETUP(".messagetracker"); -namespace storage { - -namespace distributor { +namespace storage::distributor { MessageTracker::MessageTracker(const std::string& clusterName) : _clusterName(clusterName) {} -MessageTracker::~MessageTracker() {} +MessageTracker::~MessageTracker() = default; void MessageTracker::flushQueue(MessageSender& sender) @@ -48,7 +48,4 @@ MessageTracker::finished() return _sentMessages.empty(); } - -} - } diff --git a/storage/src/vespa/storage/distributor/messagetracker.h b/storage/src/vespa/storage/distributor/messagetracker.h index 63c0be1ca93..017979c16c0 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.h +++ b/storage/src/vespa/storage/distributor/messagetracker.h @@ -1,10 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "distributormetricsset.h" #include <vespa/storage/common/messagesender.h> -#include <vespa/storageapi/messageapi/bucketcommand.h> -#include <vespa/storageapi/messageapi/bucketreply.h> +#include <vector> +#include <map> + +namespace storage::api { + class BucketCommand; + class BucketReply; +} namespace storage::distributor { diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.h b/storage/src/vespa/storage/distributor/operations/external/putoperation.h index 8beffe8b2c3..c27f2ee2266 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.h @@ -5,22 +5,21 @@ #include <vespa/storage/distributor/operations/sequenced_operation.h> #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storage/distributor/persistencemessagetracker.h> -#include <vespa/storage/distributor/operationtargetresolver.h> namespace document { class Document; } -namespace storage { -namespace lib { +namespace storage::lib { class Distribution; } -namespace api { +namespace storage::api { class CreateBucketReply; class PutCommand; } -namespace distributor { +namespace storage::distributor { class DistributorBucketSpace; +class OperationTargetList; class PutOperation : public SequencedOperation { @@ -78,5 +77,4 @@ private: DistributorBucketSpace &_bucketSpace; }; -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.h b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.h index af448c2dd55..d40924e23f0 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.h @@ -8,12 +8,11 @@ #pragma once #include <vespa/storage/distributor/operations/operation.h> +#include <map> -namespace storage { +namespace storage::api { class StatBucketCommand; } -namespace api { class StatBucketCommand; } - -namespace distributor { +namespace storage::distributor { class DistributorComponent; class DistributorBucketSpace; @@ -21,9 +20,8 @@ class DistributorBucketSpace; class StatBucketOperation : public Operation { public: - StatBucketOperation(DistributorComponent& manager, - DistributorBucketSpace &bucketSpace, - const std::shared_ptr<api::StatBucketCommand> & cmd); + StatBucketOperation(DistributorComponent& manager, DistributorBucketSpace &bucketSpace, + const std::shared_ptr<api::StatBucketCommand> & cmd); ~StatBucketOperation(); const char* getName() const override { return "statBucket"; } @@ -37,10 +35,8 @@ private: std::shared_ptr<api::StatBucketCommand> _command; - std::map<uint64_t, uint16_t> _sent; + std::map<uint64_t, uint16_t> _sent; std::map<uint16_t, std::string> _results; }; -} // distributor -} // storage - +} diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 79ffee7430c..db120880267 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -4,13 +4,12 @@ #include "getoperation.h" #include "putoperation.h" #include "updateoperation.h" -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/select/parser.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/batch.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/select/parser.h> #include <vespa/vespalib/stllike/hash_map.hpp> -#include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.twophaseupdate"); @@ -18,8 +17,7 @@ LOG_SETUP(".distributor.callback.twophaseupdate"); using namespace std::literals::string_literals; using document::BucketSpace; -namespace storage { -namespace distributor { +namespace storage::distributor { TwoPhaseUpdateOperation::TwoPhaseUpdateOperation( DistributorComponent& manager, @@ -570,5 +568,4 @@ TwoPhaseUpdateOperation::onClose(DistributorMessageSender& sender) { } } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h index f35a9dcb3ec..b4f84d76649 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h @@ -11,11 +11,10 @@ namespace document { class Document; } -namespace storage { +namespace storage { class VisitorMetricSet; } +namespace storage::lib { class ClusterState; } -class VisitorMetricSet; - -namespace distributor { +namespace storage::distributor { class DistributorComponent; class DistributorBucketSpace; @@ -181,5 +180,3 @@ private: }; } - -} diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp index 2337129e375..52c8344b820 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.cpp @@ -3,9 +3,8 @@ #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storage/distributor/idealstatemetricsset.h> -#include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storage/distributor/distributor_bucket_space_repo.h> -#include <vespa/storageapi/messageapi/maintenancecommand.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/log/log.h> LOG_SETUP(".distributor.operation"); @@ -26,17 +25,15 @@ const uint32_t IdealStateOperation::MAINTENANCE_MESSAGE_TYPES[] = }; IdealStateOperation::IdealStateOperation(const BucketAndNodes& bucketAndNodes) - : _manager(nullptr), - _bucketSpace(nullptr), - _bucketAndNodes(bucketAndNodes), - _ok(true), - _priority(255) + : _manager(nullptr), + _bucketSpace(nullptr), + _bucketAndNodes(bucketAndNodes), + _ok(true), + _priority(255) { } -IdealStateOperation::~IdealStateOperation() -{ -} +IdealStateOperation::~IdealStateOperation() = default; BucketAndNodes::BucketAndNodes(const document::Bucket &bucket, uint16_t node) : _bucket(bucket) @@ -108,8 +105,7 @@ IdealStateOperation::setCommandMeta(api::MaintenanceCommand& cmd) const { cmd.setPriority(_priority); cmd.setReason(_detailedReason); - cmd.setLoadType( - (*_manager->getLoadTypes())["maintenance"]); + cmd.setLoadType((*_manager->getLoadTypes())["maintenance"]); } std::string diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 271ac35968e..32ea695bd94 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -2,7 +2,7 @@ #include "mergeoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor_bucket_space.h> -#include <array> +#include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.merge"); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp index 1acb2dcc64b..6a87688c295 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp @@ -3,6 +3,7 @@ #include "setbucketstateoperation.h" #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor_bucket_space.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/log/log.h> LOG_SETUP(".distributor.operation.idealstate.setactive"); diff --git a/storage/src/vespa/storage/distributor/operationtargetresolver.h b/storage/src/vespa/storage/distributor/operationtargetresolver.h index 23e0fbbcba4..20666ea254c 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolver.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolver.h @@ -10,8 +10,7 @@ #include <vespa/vdslib/state/node.h> #include <vespa/vespalib/util/printable.h> -namespace storage { -namespace distributor { +namespace storage::distributor { class OperationTarget : public vespalib::AsciiPrintable { @@ -68,5 +67,4 @@ public: const document::BucketId& id) = 0; }; -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.h b/storage/src/vespa/storage/persistence/splitbitdetector.h index b3fc5bea566..6f1af6c5970 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.h +++ b/storage/src/vespa/storage/persistence/splitbitdetector.h @@ -18,6 +18,7 @@ #pragma once #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/vespalib/util/printable.h> namespace storage { diff --git a/storageapi/src/vespa/storageapi/message/state.h b/storageapi/src/vespa/storageapi/message/state.h index e8062c71d22..746d92fce6b 100644 --- a/storageapi/src/vespa/storageapi/message/state.h +++ b/storageapi/src/vespa/storageapi/message/state.h @@ -6,8 +6,7 @@ #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/vdslib/state/clusterstate.h> -namespace storage { -namespace api { +namespace storage::api { /** * @class GetNodeStateCommand @@ -90,5 +89,4 @@ public: DECLARE_STORAGEREPLY(SetSystemStateReply, onSetSystemStateReply) }; -} // api -} // storage +} diff --git a/vespajlib/src/main/java/com/yahoo/net/HostName.java b/vespajlib/src/main/java/com/yahoo/net/HostName.java index 37f7fe80246..157239e456f 100644 --- a/vespajlib/src/main/java/com/yahoo/net/HostName.java +++ b/vespajlib/src/main/java/com/yahoo/net/HostName.java @@ -27,7 +27,7 @@ public class HostName { private static final Logger logger = Logger.getLogger(HostName.class.getName()); - private static String cachedHostName = null; + private static String preferredHostName = null; /** * Return a public and fully qualified hostname for localhost that resolves to an IP address on @@ -38,14 +38,14 @@ public class HostName { * @throws RuntimeException if accessing the network or the 'hostname' command fails */ public static synchronized String getLocalhost() { - if (cachedHostName == null) { + if (preferredHostName == null) { try { - cachedHostName = getPreferredHostName(); + preferredHostName = getPreferredHostName(); } catch (Exception e) { throw new RuntimeException("Failed to find a preferred hostname", e); } } - return cachedHostName; + return preferredHostName; } private static String getPreferredHostName() throws Exception { @@ -178,4 +178,7 @@ public class HostName { } } + public static void setHostNameForTestingOnly(String hostName) { + preferredHostName = hostName; + } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java b/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java index f6237a1977a..01bf082d32f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java @@ -13,7 +13,7 @@ import java.util.Arrays; @Beta public final class DimensionSizes { - private final int[] sizes; + private final long[] sizes; private DimensionSizes(Builder builder) { this.sizes = builder.sizes; @@ -25,15 +25,15 @@ public final class DimensionSizes { * * @throws IndexOutOfBoundsException if the index is larger than the number of dimensions in this tensor minus one */ - public int size(int dimensionIndex) { return sizes[dimensionIndex]; } + public long size(int dimensionIndex) { return sizes[dimensionIndex]; } /** Returns the number of dimensions this provides the size of */ public int dimensions() { return sizes.length; } /** Returns the product of the sizes of this */ - public int totalSize() { - int productSize = 1; - for (int dimensionSize : sizes ) + public long totalSize() { + long productSize = 1; + for (long dimensionSize : sizes ) productSize *= dimensionSize; return productSize; } @@ -54,13 +54,13 @@ public final class DimensionSizes { */ public final static class Builder { - private int[] sizes; + private long[] sizes; public Builder(int dimensions) { - this.sizes = new int[dimensions]; + this.sizes = new long[dimensions]; } - public Builder set(int dimensionIndex, int size) { + public Builder set(int dimensionIndex, long size) { sizes[dimensionIndex] = size; return this; } @@ -70,7 +70,7 @@ public final class DimensionSizes { * * @throws IndexOutOfBoundsException if the index is larger than the number of dimensions in this tensor minus one */ - public int size(int dimensionIndex) { return sizes[dimensionIndex]; } + public long size(int dimensionIndex) { return sizes[dimensionIndex]; } /** Returns the number of dimensions this provides the size of */ public int dimensions() { return sizes.length; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 6b0d769de9f..7130c053e9f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -38,7 +38,7 @@ public class IndexedTensor implements Tensor { } @Override - public int size() { + public long size() { return values.length; } @@ -55,10 +55,10 @@ public class IndexedTensor implements Tensor { /** Returns an iterator over all the cells in this tensor which matches the given partial address */ // TODO: Move up to Tensor and create a mixed tensor which can implement it (and subspace iterators) efficiently public SubspaceIterator cellIterator(PartialAddress partialAddress, DimensionSizes iterationSizes) { - int[] startAddress = new int[type().dimensions().size()]; + long[] startAddress = new long[type().dimensions().size()]; List<Integer> iterateDimensions = new ArrayList<>(); for (int i = 0; i < type().dimensions().size(); i++) { - int partialAddressLabel = partialAddress.intLabel(type.dimensions().get(i).name()); + long partialAddressLabel = partialAddress.numericLabel(type.dimensions().get(i).name()); if (partialAddressLabel >= 0) // iterate at this label startAddress[i] = partialAddressLabel; else // iterate over this dimension @@ -102,8 +102,8 @@ public class IndexedTensor implements Tensor { * @param indexes the indexes into the dimensions of this. Must be one number per dimension of this * @throws IndexOutOfBoundsException if any of the indexes are out of bound or a wrong number of indexes are given */ - public double get(int ... indexes) { - return values[toValueIndex(indexes, dimensionSizes)]; + public double get(long ... indexes) { + return values[(int)toValueIndex(indexes, dimensionSizes)]; } /** Returns the value at this address, or NaN if there is no value at this address */ @@ -111,20 +111,20 @@ public class IndexedTensor implements Tensor { public double get(TensorAddress address) { // optimize for fast lookup within bounds: try { - return values[toValueIndex(address, dimensionSizes)]; + return values[(int)toValueIndex(address, dimensionSizes)]; } catch (IndexOutOfBoundsException e) { return Double.NaN; } } - private double get(int valueIndex) { return values[valueIndex]; } + private double get(long valueIndex) { return values[(int)valueIndex]; } - private static int toValueIndex(int[] indexes, DimensionSizes sizes) { + private static long toValueIndex(long[] indexes, DimensionSizes sizes) { if (indexes.length == 1) return indexes[0]; // for speed if (indexes.length == 0) return 0; // for speed - int valueIndex = 0; + long valueIndex = 0; for (int i = 0; i < indexes.length; i++) { if (indexes[i] >= sizes.size(i)) { throw new IndexOutOfBoundsException(); @@ -134,21 +134,21 @@ public class IndexedTensor implements Tensor { return valueIndex; } - private static int toValueIndex(TensorAddress address, DimensionSizes sizes) { + private static long toValueIndex(TensorAddress address, DimensionSizes sizes) { if (address.isEmpty()) return 0; - int valueIndex = 0; + long valueIndex = 0; for (int i = 0; i < address.size(); i++) { - if (address.intLabel(i) >= sizes.size(i)) { + if (address.numericLabel(i) >= sizes.size(i)) { throw new IndexOutOfBoundsException(); } - valueIndex += productOfDimensionsAfter(i, sizes) * address.intLabel(i); + valueIndex += productOfDimensionsAfter(i, sizes) * address.numericLabel(i); } return valueIndex; } - private static int productOfDimensionsAfter(int afterIndex, DimensionSizes sizes) { - int product = 1; + private static long productOfDimensionsAfter(int afterIndex, DimensionSizes sizes) { + long product = 1; for (int i = afterIndex + 1; i < sizes.dimensions(); i++) product *= sizes.size(i); return product; @@ -168,9 +168,9 @@ public class IndexedTensor implements Tensor { ImmutableMap.Builder<TensorAddress, Double> builder = new ImmutableMap.Builder<>(); Indexes indexes = Indexes.of(dimensionSizes, dimensionSizes, values.length); - for (int i = 0; i < values.length; i++) { + for (long i = 0; i < values.length; i++) { indexes.next(); - builder.put(indexes.toAddress(), values[i]); + builder.put(indexes.toAddress(), values[(int)i]); } return builder.build(); } @@ -213,7 +213,7 @@ public class IndexedTensor implements Tensor { throw new IllegalArgumentException(sizes.dimensions() + " is the wrong number of dimensions " + "for " + type); for (int i = 0; i < sizes.dimensions(); i++ ) { - Optional<Integer> size = type.dimensions().get(i).size(); + Optional<Long> size = type.dimensions().get(i).size(); if (size.isPresent() && size.get() < sizes.size(i)) throw new IllegalArgumentException("Size of dimension " + type.dimensions().get(i).name() + " is " + sizes.size(i) + @@ -223,7 +223,7 @@ public class IndexedTensor implements Tensor { return new BoundBuilder(type, sizes); } - public abstract Builder cell(double value, int ... indexes); + public abstract Builder cell(double value, long ... indexes); @Override public TensorType type() { return type; } @@ -255,12 +255,12 @@ public class IndexedTensor implements Tensor { if ( sizes.dimensions() != type.dimensions().size()) throw new IllegalArgumentException("Must have a dimension size entry for each dimension in " + type); this.sizes = sizes; - values = new double[sizes.totalSize()]; + values = new double[(int)sizes.totalSize()]; } @Override - public BoundBuilder cell(double value, int ... indexes) { - values[toValueIndex(indexes, sizes)] = value; + public BoundBuilder cell(double value, long ... indexes) { + values[(int)toValueIndex(indexes, sizes)] = value; return this; } @@ -271,7 +271,7 @@ public class IndexedTensor implements Tensor { @Override public Builder cell(TensorAddress address, double value) { - values[toValueIndex(address, sizes)] = value; + values[(int)toValueIndex(address, sizes)] = value; return this; } @@ -286,9 +286,9 @@ public class IndexedTensor implements Tensor { @Override public Builder cell(Cell cell, double value) { - int directIndex = cell.getDirectIndex(); + long directIndex = cell.getDirectIndex(); if (directIndex >= 0) // optimization - values[directIndex] = value; + values[(int)directIndex] = value; else super.cell(cell, value); return this; @@ -299,8 +299,8 @@ public class IndexedTensor implements Tensor { * This requires knowledge of the internal layout of cells in this implementation, and should therefore * probably not be used (but when it can be used it is fast). */ - public void cellByDirectIndex(int index, double value) { - values[index] = value; + public void cellByDirectIndex(long index, double value) { + values[(int)index] = value; } } @@ -326,13 +326,13 @@ public class IndexedTensor implements Tensor { return new IndexedTensor(type, new DimensionSizes.Builder(type.dimensions().size()).build(), new double[] {(Double) firstDimension.get(0) }); DimensionSizes dimensionSizes = findDimensionSizes(firstDimension); - double[] values = new double[dimensionSizes.totalSize()]; + double[] values = new double[(int)dimensionSizes.totalSize()]; fillValues(0, 0, firstDimension, dimensionSizes, values); return new IndexedTensor(type, dimensionSizes, values); } private DimensionSizes findDimensionSizes(List<Object> firstDimension) { - List<Integer> dimensionSizeList = new ArrayList<>(type.dimensions().size()); + List<Long> dimensionSizeList = new ArrayList<>(type.dimensions().size()); findDimensionSizes(0, dimensionSizeList, firstDimension); DimensionSizes.Builder b = new DimensionSizes.Builder(type.dimensions().size()); // may be longer than the list but that's correct for (int i = 0; i < b.dimensions(); i++) { @@ -343,9 +343,9 @@ public class IndexedTensor implements Tensor { } @SuppressWarnings("unchecked") - private void findDimensionSizes(int currentDimensionIndex, List<Integer> dimensionSizes, List<Object> currentDimension) { + private void findDimensionSizes(int currentDimensionIndex, List<Long> dimensionSizes, List<Object> currentDimension) { if (currentDimensionIndex == dimensionSizes.size()) - dimensionSizes.add(currentDimension.size()); + dimensionSizes.add((long)currentDimension.size()); else if (dimensionSizes.get(currentDimensionIndex) != currentDimension.size()) throw new IllegalArgumentException("Missing values in dimension " + type.dimensions().get(currentDimensionIndex) + " in " + type); @@ -356,16 +356,16 @@ public class IndexedTensor implements Tensor { } @SuppressWarnings("unchecked") - private void fillValues(int currentDimensionIndex, int offset, List<Object> currentDimension, + private void fillValues(int currentDimensionIndex, long offset, List<Object> currentDimension, DimensionSizes sizes, double[] values) { if (currentDimensionIndex < sizes.dimensions() - 1) { // recurse to next dimension - for (int i = 0; i < currentDimension.size(); i++) + for (long i = 0; i < currentDimension.size(); i++) fillValues(currentDimensionIndex + 1, offset + productOfDimensionsAfter(currentDimensionIndex, sizes) * i, - (List<Object>) currentDimension.get(i), sizes, values); + (List<Object>) currentDimension.get((int)i), sizes, values); } else { // last dimension - fill values - for (int i = 0; i < currentDimension.size(); i++) { - values[offset + i] = nullAsZero((Double)currentDimension.get(i)); // fill missing values as zero + for (long i = 0; i < currentDimension.size(); i++) { + values[(int)(offset + i)] = nullAsZero((Double)currentDimension.get((int)i)); // fill missing values as zero } } } @@ -382,9 +382,9 @@ public class IndexedTensor implements Tensor { @Override public Builder cell(TensorAddress address, double value) { - int[] indexes = new int[address.size()]; + long[] indexes = new long[address.size()]; for (int i = 0; i < address.size(); i++) { - indexes[i] = address.intLabel(i); + indexes[i] = address.numericLabel(i); } cell(value, indexes); return this; @@ -399,7 +399,7 @@ public class IndexedTensor implements Tensor { */ @SuppressWarnings("unchecked") @Override - public Builder cell(double value, int... indexes) { + public Builder cell(double value, long... indexes) { if (indexes.length != type.dimensions().size()) throw new IllegalArgumentException("Wrong number of indexes (" + indexes.length + ") for " + type); @@ -414,18 +414,18 @@ public class IndexedTensor implements Tensor { for (int dimensionIndex = 0; dimensionIndex < indexes.length; dimensionIndex++) { ensureCapacity(indexes[dimensionIndex], currentValues); if (dimensionIndex == indexes.length - 1) { // last dimension - currentValues.set(indexes[dimensionIndex], value); + currentValues.set((int)indexes[dimensionIndex], value); } else { - if (currentValues.get(indexes[dimensionIndex]) == null) - currentValues.set(indexes[dimensionIndex], new ArrayList<>()); - currentValues = (List<Object>) currentValues.get(indexes[dimensionIndex]); + if (currentValues.get((int)indexes[dimensionIndex]) == null) + currentValues.set((int)indexes[dimensionIndex], new ArrayList<>()); + currentValues = (List<Object>) currentValues.get((int)indexes[dimensionIndex]); } } return this; } /** Fill the given list with nulls if necessary to make sure it has a (possibly null) value at the given index */ - private void ensureCapacity(int index, List<Object> list) { + private void ensureCapacity(long index, List<Object> list) { while (list.size() <= index) list.add(list.size(), null); } @@ -434,7 +434,7 @@ public class IndexedTensor implements Tensor { private final class CellIterator implements Iterator<Cell> { - private int count = 0; + private long count = 0; private final Indexes indexes = Indexes.of(dimensionSizes, dimensionSizes, values.length); private final LazyCell reusedCell = new LazyCell(indexes, Double.NaN); @@ -456,7 +456,7 @@ public class IndexedTensor implements Tensor { private final class ValueIterator implements Iterator<Double> { - private int count = 0; + private long count = 0; @Override public boolean hasNext() { @@ -466,7 +466,7 @@ public class IndexedTensor implements Tensor { @Override public Double next() { try { - return values[count++]; + return values[(int)count++]; } catch (IndexOutOfBoundsException e) { throw new NoSuchElementException("No element at position " + count); @@ -479,7 +479,7 @@ public class IndexedTensor implements Tensor { private final Indexes superindexes; - /** Those indexes this should iterate over */ + /** The indexes this should iterate over */ private final List<Integer> subdimensionIndexes; /** @@ -488,7 +488,7 @@ public class IndexedTensor implements Tensor { */ private final DimensionSizes iterateSizes; - private int count = 0; + private long count = 0; private SuperspaceIterator(Set<String> superdimensionNames, DimensionSizes iterateSizes) { this.iterateSizes = iterateSizes; @@ -533,11 +533,11 @@ public class IndexedTensor implements Tensor { * This may be any subset of the dimensions given by address and dimensionSizes. */ private final List<Integer> iterateDimensions; - private final int[] address; + private final long[] address; private final DimensionSizes iterateSizes; private Indexes indexes; - private int count = 0; + private long count = 0; /** A lazy cell for reuse */ private final LazyCell reusedCell; @@ -554,7 +554,7 @@ public class IndexedTensor implements Tensor { * This is treated as immutable. * @param address the address of the first cell of this subspace. */ - private SubspaceIterator(List<Integer> iterateDimensions, int[] address, DimensionSizes iterateSizes) { + private SubspaceIterator(List<Integer> iterateDimensions, long[] address, DimensionSizes iterateSizes) { this.iterateDimensions = iterateDimensions; this.address = address; this.iterateSizes = iterateSizes; @@ -563,7 +563,7 @@ public class IndexedTensor implements Tensor { } /** Returns the total number of cells in this subspace */ - public int size() { + public long size() { return indexes.size(); } @@ -605,7 +605,7 @@ public class IndexedTensor implements Tensor { } @Override - int getDirectIndex() { return indexes.toIterationValueIndex(); } + long getDirectIndex() { return indexes.toIterationValueIndex(); } @Override public TensorAddress getKey() { @@ -630,7 +630,7 @@ public class IndexedTensor implements Tensor { private final DimensionSizes iterationSizes; - protected final int[] indexes; + protected final long[] indexes; public static Indexes of(DimensionSizes sizes) { return of(sizes, sizes); @@ -640,7 +640,7 @@ public class IndexedTensor implements Tensor { return of(sourceSizes, iterateSizes, completeIterationOrder(iterateSizes.dimensions())); } - private static Indexes of(DimensionSizes sourceSizes, DimensionSizes iterateSizes, int size) { + private static Indexes of(DimensionSizes sourceSizes, DimensionSizes iterateSizes, long size) { return of(sourceSizes, iterateSizes, completeIterationOrder(iterateSizes.dimensions()), size); } @@ -648,15 +648,15 @@ public class IndexedTensor implements Tensor { return of(sourceSizes, iterateSizes, iterateDimensions, computeSize(iterateSizes, iterateDimensions)); } - private static Indexes of(DimensionSizes sourceSizes, DimensionSizes iterateSizes, List<Integer> iterateDimensions, int size) { - return of(sourceSizes, iterateSizes, iterateDimensions, new int[iterateSizes.dimensions()], size); + private static Indexes of(DimensionSizes sourceSizes, DimensionSizes iterateSizes, List<Integer> iterateDimensions, long size) { + return of(sourceSizes, iterateSizes, iterateDimensions, new long[iterateSizes.dimensions()], size); } - private static Indexes of(DimensionSizes sourceSizes, DimensionSizes iterateSizes, List<Integer> iterateDimensions, int[] initialIndexes) { + private static Indexes of(DimensionSizes sourceSizes, DimensionSizes iterateSizes, List<Integer> iterateDimensions, long[] initialIndexes) { return of(sourceSizes, iterateSizes, iterateDimensions, initialIndexes, computeSize(iterateSizes, iterateDimensions)); } - private static Indexes of(DimensionSizes sourceSizes, DimensionSizes iterateSizes, List<Integer> iterateDimensions, int[] initialIndexes, int size) { + private static Indexes of(DimensionSizes sourceSizes, DimensionSizes iterateSizes, List<Integer> iterateDimensions, long[] initialIndexes, long size) { if (size == 0) { return new EmptyIndexes(sourceSizes, iterateSizes, initialIndexes); // we're told explicitly there are truly no values available } @@ -684,14 +684,14 @@ public class IndexedTensor implements Tensor { return iterationDimensions; } - private Indexes(DimensionSizes sourceSizes, DimensionSizes iterationSizes, int[] indexes) { + private Indexes(DimensionSizes sourceSizes, DimensionSizes iterationSizes, long[] indexes) { this.sourceSizes = sourceSizes; this.iterationSizes = iterationSizes; this.indexes = indexes; } - private static int computeSize(DimensionSizes sizes, List<Integer> iterateDimensions) { - int size = 1; + private static long computeSize(DimensionSizes sizes, List<Integer> iterateDimensions) { + long size = 1; for (int iterateDimension : iterateDimensions) size *= sizes.size(iterateDimension); return size; @@ -702,25 +702,25 @@ public class IndexedTensor implements Tensor { return TensorAddress.of(indexes); } - public int[] indexesCopy() { + public long[] indexesCopy() { return Arrays.copyOf(indexes, indexes.length); } /** Returns a copy of the indexes of this which must not be modified */ - public int[] indexesForReading() { return indexes; } + public long[] indexesForReading() { return indexes; } - int toSourceValueIndex() { + long toSourceValueIndex() { return IndexedTensor.toValueIndex(indexes, sourceSizes); } - int toIterationValueIndex() { return IndexedTensor.toValueIndex(indexes, iterationSizes); } + long toIterationValueIndex() { return IndexedTensor.toValueIndex(indexes, iterationSizes); } DimensionSizes dimensionSizes() { return iterationSizes; } /** Returns an immutable list containing a copy of the indexes in this */ - public List<Integer> toList() { - ImmutableList.Builder<Integer> builder = new ImmutableList.Builder<>(); - for (int index : indexes) + public List<Long> toList() { + ImmutableList.Builder<Long> builder = new ImmutableList.Builder<>(); + for (long index : indexes) builder.add(index); return builder.build(); } @@ -730,7 +730,7 @@ public class IndexedTensor implements Tensor { return "indexes " + Arrays.toString(indexes); } - public abstract int size(); + public abstract long size(); public abstract void next(); @@ -738,12 +738,12 @@ public class IndexedTensor implements Tensor { private final static class EmptyIndexes extends Indexes { - private EmptyIndexes(DimensionSizes sourceSizes, DimensionSizes iterateSizes, int[] indexes) { + private EmptyIndexes(DimensionSizes sourceSizes, DimensionSizes iterateSizes, long[] indexes) { super(sourceSizes, iterateSizes, indexes); } @Override - public int size() { return 0; } + public long size() { return 0; } @Override public void next() {} @@ -752,12 +752,12 @@ public class IndexedTensor implements Tensor { private final static class SingleValueIndexes extends Indexes { - private SingleValueIndexes(DimensionSizes sourceSizes, DimensionSizes iterateSizes, int[] indexes) { + private SingleValueIndexes(DimensionSizes sourceSizes, DimensionSizes iterateSizes, long[] indexes) { super(sourceSizes, iterateSizes, indexes); } @Override - public int size() { return 1; } + public long size() { return 1; } @Override public void next() {} @@ -766,11 +766,11 @@ public class IndexedTensor implements Tensor { private static class MultiDimensionIndexes extends Indexes { - private final int size; + private final long size; private final List<Integer> iterateDimensions; - private MultiDimensionIndexes(DimensionSizes sourceSizes, DimensionSizes iterateSizes, List<Integer> iterateDimensions, int[] initialIndexes, int size) { + private MultiDimensionIndexes(DimensionSizes sourceSizes, DimensionSizes iterateSizes, List<Integer> iterateDimensions, long[] initialIndexes, long size) { super(sourceSizes, iterateSizes, initialIndexes); this.iterateDimensions = iterateDimensions; this.size = size; @@ -781,7 +781,7 @@ public class IndexedTensor implements Tensor { /** Returns the number of values this will iterate over - i.e the product if the iterating dimension sizes */ @Override - public int size() { + public long size() { return size; } @@ -806,36 +806,38 @@ public class IndexedTensor implements Tensor { /** In this case we can reuse the source index computation for the iteration index */ private final static class EqualSizeMultiDimensionIndexes extends MultiDimensionIndexes { - private int lastComputedSourceValueIndex = -1; + private long lastComputedSourceValueIndex = -1; - private EqualSizeMultiDimensionIndexes(DimensionSizes sizes, List<Integer> iterateDimensions, int[] initialIndexes, int size) { + private EqualSizeMultiDimensionIndexes(DimensionSizes sizes, List<Integer> iterateDimensions, long[] initialIndexes, long size) { super(sizes, sizes, iterateDimensions, initialIndexes, size); } - int toSourceValueIndex() { + @Override + long toSourceValueIndex() { return lastComputedSourceValueIndex = super.toSourceValueIndex(); } // NOTE: We assume the source index always gets computed first. Otherwise using this will produce a runtime exception - int toIterationValueIndex() { return lastComputedSourceValueIndex; } + @Override + long toIterationValueIndex() { return lastComputedSourceValueIndex; } } /** In this case we can keep track of indexes using a step instead of using the more elaborate computation */ private final static class SingleDimensionIndexes extends Indexes { - private final int size; + private final long size; private final int iterateDimension; /** Maintain this directly as an optimization for 1-d iteration */ - private int currentSourceValueIndex, currentIterationValueIndex; + private long currentSourceValueIndex, currentIterationValueIndex; /** The iteration step in the value index space */ - private final int sourceStep, iterationStep; + private final long sourceStep, iterationStep; private SingleDimensionIndexes(DimensionSizes sourceSizes, DimensionSizes iterateSizes, - int iterateDimension, int[] initialIndexes, int size) { + int iterateDimension, long[] initialIndexes, long size) { super(sourceSizes, iterateSizes, initialIndexes); this.iterateDimension = iterateDimension; this.size = size; @@ -850,7 +852,7 @@ public class IndexedTensor implements Tensor { /** Returns the number of values this will iterate over - i.e the product if the iterating dimension sizes */ @Override - public int size() { + public long size() { return size; } @@ -868,28 +870,28 @@ public class IndexedTensor implements Tensor { } @Override - int toSourceValueIndex() { return currentSourceValueIndex; } + long toSourceValueIndex() { return currentSourceValueIndex; } @Override - int toIterationValueIndex() { return currentIterationValueIndex; } + long toIterationValueIndex() { return currentIterationValueIndex; } } /** In this case we only need to keep track of one index */ private final static class EqualSizeSingleDimensionIndexes extends Indexes { - private final int size; + private final long size; private final int iterateDimension; /** Maintain this directly as an optimization for 1-d iteration */ - private int currentValueIndex; + private long currentValueIndex; /** The iteration step in the value index space */ - private final int step; + private final long step; private EqualSizeSingleDimensionIndexes(DimensionSizes sizes, - int iterateDimension, int[] initialIndexes, int size) { + int iterateDimension, long[] initialIndexes, long size) { super(sizes, sizes, initialIndexes); this.iterateDimension = iterateDimension; this.size = size; @@ -902,7 +904,7 @@ public class IndexedTensor implements Tensor { /** Returns the number of values this will iterate over - i.e the product if the iterating dimension sizes */ @Override - public int size() { + public long size() { return size; } @@ -919,10 +921,10 @@ public class IndexedTensor implements Tensor { } @Override - int toSourceValueIndex() { return currentValueIndex; } + long toSourceValueIndex() { return currentValueIndex; } @Override - int toIterationValueIndex() { return currentValueIndex; } + long toIterationValueIndex() { return currentValueIndex; } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java index aba61478e69..15993072c37 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java @@ -29,7 +29,7 @@ public class MappedTensor implements Tensor { public TensorType type() { return type; } @Override - public int size() { return cells.size(); } + public long size() { return cells.size(); } @Override public double get(TensorAddress address) { return cells.getOrDefault(address, Double.NaN); } @@ -80,7 +80,7 @@ public class MappedTensor implements Tensor { } @Override - public Builder cell(double value, int... labels) { + public Builder cell(double value, long... labels) { cells.put(TensorAddress.of(labels), value); return this; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 9a751e078e0..0c9ed769c0d 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -47,13 +47,13 @@ public class MixedTensor implements Tensor { /** Returns the size of the tensor measured in number of cells */ @Override - public int size() { return cells.size(); } + public long size() { return cells.size(); } /** Returns the value at the given address */ @Override public double get(TensorAddress address) { - int cellIndex = index.indexOf(address); - Cell cell = cells.get(cellIndex); + long cellIndex = index.indexOf(address); + Cell cell = cells.get((int)cellIndex); if (!address.equals(cell.getKey())) { throw new IllegalStateException("Unable to find correct cell by direct index."); } @@ -113,7 +113,7 @@ public class MixedTensor implements Tensor { } /** Returns the size of dense subspaces */ - public int denseSubspaceSize() { + public long denseSubspaceSize() { return index.denseSubspaceSize(); } @@ -148,7 +148,7 @@ public class MixedTensor implements Tensor { } @Override - public Tensor.Builder cell(double value, int... labels) { + public Tensor.Builder cell(double value, long... labels) { throw new UnsupportedOperationException("Not implemented."); } @@ -179,13 +179,13 @@ public class MixedTensor implements Tensor { index = indexBuilder.index(); } - public int denseSubspaceSize() { + public long denseSubspaceSize() { return index.denseSubspaceSize(); } private double[] denseSubspace(TensorAddress sparsePartial) { if (!denseSubspaceMap.containsKey(sparsePartial)) { - denseSubspaceMap.put(sparsePartial, new double[denseSubspaceSize()]); + denseSubspaceMap.put(sparsePartial, new double[(int)denseSubspaceSize()]); } return denseSubspaceMap.get(sparsePartial); } @@ -193,21 +193,21 @@ public class MixedTensor implements Tensor { @Override public Tensor.Builder cell(TensorAddress address, double value) { TensorAddress sparsePart = index.sparsePartialAddress(address); - int denseOffset = index.denseOffset(address); + long denseOffset = index.denseOffset(address); double[] denseSubspace = denseSubspace(sparsePart); - denseSubspace[denseOffset] = value; + denseSubspace[(int)denseOffset] = value; return this; } public Tensor.Builder block(TensorAddress sparsePart, double[] values) { double[] denseSubspace = denseSubspace(sparsePart); - System.arraycopy(values, 0, denseSubspace, 0, denseSubspaceSize()); + System.arraycopy(values, 0, denseSubspace, 0, (int)denseSubspaceSize()); return this; } @Override public MixedTensor build() { - int count = 0; + long count = 0; ImmutableList.Builder<Cell> builder = new ImmutableList.Builder<>(); for (Map.Entry<TensorAddress, double[]> entry : denseSubspaceMap.entrySet()) { @@ -215,9 +215,9 @@ public class MixedTensor implements Tensor { indexBuilder.put(sparsePart, count); double[] denseSubspace = entry.getValue(); - for (int offset = 0; offset < denseSubspace.length; ++offset) { + for (long offset = 0; offset < denseSubspace.length; ++offset) { TensorAddress cellAddress = index.addressOf(sparsePart, offset); - double value = denseSubspace[offset]; + double value = denseSubspace[(int)offset]; builder.add(new Cell(cellAddress, value)); count++; } @@ -239,12 +239,12 @@ public class MixedTensor implements Tensor { public static class UnboundBuilder extends Builder { private Map<TensorAddress, Double> cells; - private final int[] dimensionBounds; + private final long[] dimensionBounds; private UnboundBuilder(TensorType type) { super(type); cells = new HashMap<>(); - dimensionBounds = new int[type.dimensions().size()]; + dimensionBounds = new long[type.dimensions().size()]; } @Override @@ -268,7 +268,7 @@ public class MixedTensor implements Tensor { for (int i = 0; i < type.dimensions().size(); ++i) { TensorType.Dimension dimension = type.dimensions().get(i); if (dimension.isIndexed()) { - dimensionBounds[i] = Math.max(address.intLabel(i), dimensionBounds[i]); + dimensionBounds[i] = Math.max(address.numericLabel(i), dimensionBounds[i]); } } } @@ -280,7 +280,7 @@ public class MixedTensor implements Tensor { if (!dimension.isIndexed()) { typeBuilder.mapped(dimension.name()); } else { - int size = dimension.size().orElse(dimensionBounds[i] + 1); + long size = dimension.size().orElse(dimensionBounds[i] + 1); typeBuilder.indexed(dimension.name(), size); } } @@ -303,8 +303,8 @@ public class MixedTensor implements Tensor { private final List<TensorType.Dimension> mappedDimensions; private final List<TensorType.Dimension> indexedDimensions; - private ImmutableMap<TensorAddress, Integer> sparseMap; - private int denseSubspaceSize = -1; + private ImmutableMap<TensorAddress, Long> sparseMap; + private long denseSubspaceSize = -1; private Index(TensorType type) { this.type = type; @@ -314,26 +314,27 @@ public class MixedTensor implements Tensor { this.denseType = createPartialType(indexedDimensions); } - public int indexOf(TensorAddress address) { + public long indexOf(TensorAddress address) { TensorAddress sparsePart = sparsePartialAddress(address); - if (!sparseMap.containsKey(sparsePart)) { + if ( ! sparseMap.containsKey(sparsePart)) { throw new IllegalArgumentException("Address not found"); } - int base = sparseMap.get(sparsePart); - int offset = denseOffset(address); + long base = sparseMap.get(sparsePart); + long offset = denseOffset(address); return base + offset; } public static class Builder { + private final Index index; - private final ImmutableMap.Builder<TensorAddress, Integer> builder; + private final ImmutableMap.Builder<TensorAddress, Long> builder; public Builder(TensorType type) { index = new Index(type); builder = new ImmutableMap.Builder<>(); } - public void put(TensorAddress address, int index) { + public void put(TensorAddress address, long index) { builder.put(address, index); } @@ -347,7 +348,7 @@ public class MixedTensor implements Tensor { } } - public int denseSubspaceSize() { + public long denseSubspaceSize() { if (denseSubspaceSize == -1) { denseSubspaceSize = 1; for (int i = 0; i < type.dimensions().size(); ++i) { @@ -375,13 +376,13 @@ public class MixedTensor implements Tensor { return builder.build(); } - private int denseOffset(TensorAddress address) { - int innerSize = 1; - int offset = 0; + private long denseOffset(TensorAddress address) { + long innerSize = 1; + long offset = 0; for (int i = type.dimensions().size(); --i >= 0; ) { TensorType.Dimension dimension = type.dimensions().get(i); if (dimension.isIndexed()) { - int label = address.intLabel(i); + long label = address.numericLabel(i); offset += label * innerSize; innerSize *= dimension.size().orElseThrow(() -> new IllegalArgumentException("Unknown size of indexed dimension.")); @@ -390,18 +391,18 @@ public class MixedTensor implements Tensor { return offset; } - private TensorAddress denseOffsetToAddress(int denseOffset) { + private TensorAddress denseOffsetToAddress(long denseOffset) { if (denseOffset < 0 || denseOffset > denseSubspaceSize) { throw new IllegalArgumentException("Offset out of bounds"); } - int restSize = denseOffset; - int innerSize = denseSubspaceSize; - int[] labels = new int[indexedDimensions.size()]; + long restSize = denseOffset; + long innerSize = denseSubspaceSize; + long[] labels = new long[indexedDimensions.size()]; for (int i = 0; i < labels.length; ++i) { TensorType.Dimension dimension = indexedDimensions.get(i); - int dimensionSize = dimension.size().orElseThrow(() -> + long dimensionSize = dimension.size().orElseThrow(() -> new IllegalArgumentException("Unknown size of indexed dimension.")); innerSize /= dimensionSize; @@ -411,7 +412,7 @@ public class MixedTensor implements Tensor { return TensorAddress.of(labels); } - private TensorAddress addressOf(TensorAddress sparsePart, int denseOffset) { + private TensorAddress addressOf(TensorAddress sparsePart, long denseOffset) { TensorAddress densePart = denseOffsetToAddress(denseOffset); String[] labels = new String[type.dimensions().size()]; int mappedIndex = 0; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/PartialAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/PartialAddress.java index e3398850373..23ef0772aea 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/PartialAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/PartialAddress.java @@ -6,11 +6,11 @@ import com.google.common.annotations.Beta; /** * An address to a subset of a tensors' cells, specifying a label for some but not necessarily all of the tensors * dimensions. - * + * * @author bratseth */ -// Implementation notes: -// - These are created in inner (though not inner-most) loops so they are implemented with minimal allocation. +// Implementation notes: +// - These are created in inner (though not inner-most) loops so they are implemented with minimal allocation. // We also avoid non-essential error checking. // - We can add support for string labels later without breaking the API @Beta @@ -19,7 +19,7 @@ public class PartialAddress { // Two arrays which contains corresponding dimension=label pairs. // The sizes of these are always equal. private final String[] dimensionNames; - private final int[] labels; + private final long[] labels; private PartialAddress(Builder builder) { this.dimensionNames = builder.dimensionNames; @@ -27,36 +27,36 @@ public class PartialAddress { builder.dimensionNames = null; // invalidate builder to safely take over array ownership builder.labels = null; } - + /** Returns the int label of this dimension, or -1 if no label is specified for it */ - int intLabel(String dimensionName) { + long numericLabel(String dimensionName) { for (int i = 0; i < dimensionNames.length; i++) if (dimensionNames[i].equals(dimensionName)) return labels[i]; return -1; } - + public static class Builder { private String[] dimensionNames; - private int[] labels; + private long[] labels; private int index = 0; - + public Builder(int size) { dimensionNames = new String[size]; - labels = new int[size]; + labels = new long[size]; } - - public void add(String dimensionName, int label) { + + public void add(String dimensionName, long label) { dimensionNames[index] = dimensionName; labels[index] = label; index++; } - + public PartialAddress build() { return new PartialAddress(this); } - + } - + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 1b60e01cf7e..0c948f1fbee 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -59,7 +59,7 @@ public interface Tensor { default boolean isEmpty() { return size() == 0; } /** Returns the number of cells in this */ - int size(); + long size(); /** Returns the value of a cell, or NaN if this cell does not exist/have no value */ double get(TensorAddress address); @@ -124,7 +124,7 @@ public interface Tensor { return new Rename(new ConstantTensor(this), fromDimensions, toDimensions).evaluate(); } - static Tensor generate(TensorType type, Function<List<Integer>, Double> valueSupplier) { + static Tensor generate(TensorType type, Function<List<Long>, Double> valueSupplier) { return new Generate(type, valueSupplier).evaluate(); } @@ -333,7 +333,7 @@ public interface Tensor { * This is for optimizations mapping between tensors where this is possible without creating a * TensorAddress. */ - int getDirectIndex() { return -1; } + long getDirectIndex() { return -1; } @Override public Double getValue() { return value; } @@ -396,7 +396,7 @@ public interface Tensor { Builder cell(TensorAddress address, double value); /** Add a cell */ - Builder cell(double value, int ... labels); + Builder cell(double value, long ... labels); /** * Add a cell @@ -425,7 +425,7 @@ public interface Tensor { return this; } - public CellBuilder label(String dimension, int label) { + public CellBuilder label(String dimension, long label) { return label(dimension, String.valueOf(label)); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index ff1202463f2..38553497478 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -2,16 +2,10 @@ package com.yahoo.tensor; import com.google.common.annotations.Beta; -import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * An immutable address to a tensor cell. This simply supplies a value to each dimension @@ -26,8 +20,8 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { return new StringTensorAddress(labels); } - public static TensorAddress of(int ... labels) { - return new IntTensorAddress(labels); + public static TensorAddress of(long ... labels) { + return new NumericTensorAddress(labels); } /** Returns the number of labels in this */ @@ -41,14 +35,14 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { public abstract String label(int i); /** - * Returns the i'th label in this as an int. - * Prefer this if you know that this is an integer address, but not otherwise. + * Returns the i'th label in this as a long. + * Prefer this if you know that this is a numeric address, but not otherwise. * * @throws IllegalArgumentException if there is no label at this index */ - public abstract int intLabel(int i); + public abstract long numericLabel(int i); - public abstract TensorAddress withLabel(int labelIndex, int label); + public abstract TensorAddress withLabel(int labelIndex, long label); public final boolean isEmpty() { return size() == 0; } @@ -110,17 +104,17 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { public String label(int i) { return labels[i]; } @Override - public int intLabel(int i) { + public long numericLabel(int i) { try { - return Integer.parseInt(labels[i]); + return Long.parseLong(labels[i]); } catch (NumberFormatException e) { - throw new IllegalArgumentException("Expected an int label in " + this + " at position " + i); + throw new IllegalArgumentException("Expected a long label in " + this + " at position " + i); } } @Override - public TensorAddress withLabel(int index, int label) { + public TensorAddress withLabel(int index, long label) { String[] labels = Arrays.copyOf(this.labels, this.labels.length); labels[index] = String.valueOf(label); return new StringTensorAddress(labels); @@ -133,11 +127,11 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { } - private static final class IntTensorAddress extends TensorAddress { + private static final class NumericTensorAddress extends TensorAddress { - private final int[] labels; + private final long[] labels; - private IntTensorAddress(int[] labels) { + private NumericTensorAddress(long[] labels) { this.labels = Arrays.copyOf(labels, labels.length); } @@ -148,13 +142,13 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { public String label(int i) { return String.valueOf(labels[i]); } @Override - public int intLabel(int i) { return labels[i]; } + public long numericLabel(int i) { return labels[i]; } @Override - public TensorAddress withLabel(int index, int label) { - int[] labels = Arrays.copyOf(this.labels, this.labels.length); + public TensorAddress withLabel(int index, long label) { + long[] labels = Arrays.copyOf(this.labels, this.labels.length); labels[index] = label; - return new IntTensorAddress(labels); + return new NumericTensorAddress(labels); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index 914d853aeca..b396f831de0 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -139,7 +139,7 @@ public class TensorType { public final String name() { return name; } /** Returns the size of this dimension if it is bound, empty otherwise */ - public abstract Optional<Integer> size(); + public abstract Optional<Long> size(); public abstract Type type(); @@ -189,7 +189,7 @@ public class TensorType { return this.name.compareTo(other.name); } - public static Dimension indexed(String name, int size) { + public static Dimension indexed(String name, long size) { return new IndexedBoundDimension(name, size); } @@ -197,17 +197,19 @@ public class TensorType { public static class IndexedBoundDimension extends TensorType.Dimension { - private final Integer size; + private final Long size; - private IndexedBoundDimension(String name, int size) { + private IndexedBoundDimension(String name, long size) { super(name); if (size < 1) throw new IllegalArgumentException("Size of bound dimension '" + name + "' must be at least 1"); + if (size > Integer.MAX_VALUE) + throw new IllegalArgumentException("Size of bound dimension '" + name + "' cannot be larger than " + Integer.MAX_VALUE); this.size = size; } @Override - public Optional<Integer> size() { return Optional.of(size); } + public Optional<Long> size() { return Optional.of(size); } @Override public Type type() { return Type.indexedBound; } @@ -248,7 +250,7 @@ public class TensorType { } @Override - public Optional<Integer> size() { return Optional.empty(); } + public Optional<Long> size() { return Optional.empty(); } @Override public Type type() { return Type.indexedUnbound; } @@ -269,7 +271,7 @@ public class TensorType { } @Override - public Optional<Integer> size() { return Optional.empty(); } + public Optional<Long> size() { return Optional.empty(); } @Override public Type type() { return Type.mapped; } @@ -357,7 +359,7 @@ public class TensorType { * * @throws IllegalArgumentException if the dimension is already present */ - public Builder indexed(String name, int size) { return add(new IndexedBoundDimension(name, size)); } + public Builder indexed(String name, long size) { return add(new IndexedBoundDimension(name, size)); } /** * Adds an unbound indexed dimension to this diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java index faa0ca36cb6..d4affe0ef9b 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java @@ -67,7 +67,7 @@ public class Concat extends PrimitiveTensorFunction { DimensionSizes concatSize = concatSize(concatType, aIndexed, bIndexed, dimension); Tensor.Builder builder = Tensor.Builder.of(concatType, concatSize); - int aDimensionLength = aIndexed.type().indexOfDimension(dimension).map(d -> aIndexed.dimensionSizes().size(d)).orElseThrow(RuntimeException::new); + long aDimensionLength = aIndexed.type().indexOfDimension(dimension).map(d -> aIndexed.dimensionSizes().size(d)).orElseThrow(RuntimeException::new); int[] aToIndexes = mapIndexes(a.type(), concatType); int[] bToIndexes = mapIndexes(b.type(), concatType); concatenateTo(aIndexed, bIndexed, aDimensionLength, concatType, aToIndexes, bToIndexes, builder); @@ -75,7 +75,7 @@ public class Concat extends PrimitiveTensorFunction { return builder.build(); } - private void concatenateTo(IndexedTensor a, IndexedTensor b, int offset, TensorType concatType, + private void concatenateTo(IndexedTensor a, IndexedTensor b, long offset, TensorType concatType, int[] aToIndexes, int[] bToIndexes, Tensor.Builder builder) { Set<String> otherADimensions = a.type().dimensionNames().stream().filter(d -> !d.equals(dimension)).collect(Collectors.toSet()); for (Iterator<IndexedTensor.SubspaceIterator> ia = a.subspaceIterator(otherADimensions); ia.hasNext();) { @@ -129,8 +129,8 @@ public class Concat extends PrimitiveTensorFunction { DimensionSizes.Builder concatSizes = new DimensionSizes.Builder(concatType.dimensions().size()); for (int i = 0; i < concatSizes.dimensions(); i++) { String currentDimension = concatType.dimensions().get(i).name(); - int aSize = a.type().indexOfDimension(currentDimension).map(d -> a.dimensionSizes().size(d)).orElse(0); - int bSize = b.type().indexOfDimension(currentDimension).map(d -> b.dimensionSizes().size(d)).orElse(0); + long aSize = a.type().indexOfDimension(currentDimension).map(d -> a.dimensionSizes().size(d)).orElse(0L); + long bSize = b.type().indexOfDimension(currentDimension).map(d -> b.dimensionSizes().size(d)).orElse(0L); if (currentDimension.equals(concatDimension)) concatSizes.set(i, aSize + bSize); else if (aSize != 0 && bSize != 0 && aSize!=bSize ) @@ -148,8 +148,8 @@ public class Concat extends PrimitiveTensorFunction { * (in some other dimension than the concat dimension) */ private TensorAddress combineAddresses(TensorAddress a, int[] aToIndexes, TensorAddress b, int[] bToIndexes, - TensorType concatType, int concatOffset, String concatDimension) { - int[] combinedLabels = new int[concatType.dimensions().size()]; + TensorType concatType, long concatOffset, String concatDimension) { + long[] combinedLabels = new long[concatType.dimensions().size()]; Arrays.fill(combinedLabels, -1); int concatDimensionIndex = concatType.indexOfDimension(concatDimension).get(); mapContent(a, combinedLabels, aToIndexes, concatDimensionIndex, concatOffset); // note: This sets a nonsensical value in the concat dimension @@ -179,15 +179,15 @@ public class Concat extends PrimitiveTensorFunction { * @return true if the mapping was successful, false if one of the destination positions was * occupied by a different value */ - private boolean mapContent(TensorAddress from, int[] to, int[] indexMap, int concatDimension, int concatOffset) { + private boolean mapContent(TensorAddress from, long[] to, int[] indexMap, int concatDimension, long concatOffset) { for (int i = 0; i < from.size(); i++) { int toIndex = indexMap[i]; if (concatDimension == toIndex) { - to[toIndex] = from.intLabel(i) + concatOffset; + to[toIndex] = from.numericLabel(i) + concatOffset; } else { - if (to[toIndex] != -1 && to[toIndex] != from.intLabel(i)) return false; - to[toIndex] = from.intLabel(i); + if (to[toIndex] != -1 && to[toIndex] != from.numericLabel(i)) return false; + to[toIndex] = from.numericLabel(i); } } return true; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java index c75d8ee4753..653be8dacf0 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java @@ -17,7 +17,7 @@ import java.util.stream.Stream; public class Diag extends CompositeTensorFunction { private final TensorType type; - private final Function<List<Integer>, Double> diagFunction; + private final Function<List<Long>, Double> diagFunction; public Diag(TensorType type) { this.type = type; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java index e42d25197e2..ef2770c04f5 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java @@ -22,17 +22,17 @@ import java.util.function.Function; public class Generate extends PrimitiveTensorFunction { private final TensorType type; - private final Function<List<Integer>, Double> generator; + private final Function<List<Long>, Double> generator; /** * Creates a generated tensor * * @param type the type of the tensor - * @param generator the function generating values from a list of ints specifying the indexes of the + * @param generator the function generating values from a list of numbers specifying the indexes of the * tensor cell which will receive the value * @throws IllegalArgumentException if any of the tensor dimensions are not indexed bound */ - public Generate(TensorType type, Function<List<Integer>, Double> generator) { + public Generate(TensorType type, Function<List<Long>, Double> generator) { Objects.requireNonNull(type, "The argument tensor type cannot be null"); Objects.requireNonNull(generator, "The argument function cannot be null"); validateType(type); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java index ff887e3e9a6..174a8e4c435 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java @@ -56,8 +56,8 @@ public class Join extends PrimitiveTensorFunction { if (aDim.name().equals(bDim.name())) { // include if (aDim.isIndexed() && bDim.isIndexed()) { if (aDim.size().isPresent() || bDim.size().isPresent()) - typeBuilder.indexed(aDim.name(), Math.min(aDim.size().orElse(Integer.MAX_VALUE), - bDim.size().orElse(Integer.MAX_VALUE))); + typeBuilder.indexed(aDim.name(), Math.min(aDim.size().orElse(Long.MAX_VALUE), + bDim.size().orElse(Long.MAX_VALUE))); else typeBuilder.indexed(aDim.name()); } @@ -118,11 +118,11 @@ public class Join extends PrimitiveTensorFunction { } private Tensor indexedVectorJoin(IndexedTensor a, IndexedTensor b, TensorType type) { - int joinedLength = Math.min(a.dimensionSizes().size(0), b.dimensionSizes().size(0)); + long joinedRank = Math.min(a.dimensionSizes().size(0), b.dimensionSizes().size(0)); Iterator<Double> aIterator = a.valueIterator(); Iterator<Double> bIterator = b.valueIterator(); - IndexedTensor.Builder builder = IndexedTensor.Builder.of(type, new DimensionSizes.Builder(1).set(0, joinedLength).build()); - for (int i = 0; i < joinedLength; i++) + IndexedTensor.Builder builder = IndexedTensor.Builder.of(type, new DimensionSizes.Builder(1).set(0, joinedRank).build()); + for (int i = 0; i < joinedRank; i++) builder.cell(combinator.applyAsDouble(aIterator.next(), bIterator.next()), i); return builder.build(); } @@ -169,10 +169,10 @@ public class Join extends PrimitiveTensorFunction { return builder.build(); } - private void joinSubspaces(Iterator<Double> subspace, int subspaceSize, - Iterator<Tensor.Cell> superspace, int superspaceSize, + private void joinSubspaces(Iterator<Double> subspace, long subspaceSize, + Iterator<Tensor.Cell> superspace, long superspaceSize, boolean reversedArgumentOrder, IndexedTensor.Builder builder) { - int joinedLength = Math.min(subspaceSize, superspaceSize); + long joinedLength = Math.min(subspaceSize, superspaceSize); if (reversedArgumentOrder) { for (int i = 0; i < joinedLength; i++) { Tensor.Cell supercell = superspace.next(); @@ -281,7 +281,7 @@ public class Join extends PrimitiveTensorFunction { PartialAddress.Builder builder = new PartialAddress.Builder(retainDimensions.size()); for (int i = 0; i < addressType.dimensions().size(); i++) if (retainDimensions.contains(addressType.dimensions().get(i).name())) - builder.add(addressType.dimensions().get(i).name(), address.intLabel(i)); + builder.add(addressType.dimensions().get(i).name(), address.numericLabel(i)); return builder.build(); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java index a56f82b026a..8e7f4e4c773 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java @@ -18,7 +18,7 @@ import java.util.stream.Stream; public class Range extends CompositeTensorFunction { private final TensorType type; - private final Function<List<Integer>, Double> rangeFunction; + private final Function<List<Long>, Double> rangeFunction; public Range(TensorType type) { this.type = type; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java index fb5029fbfd6..f1dadba2a29 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java @@ -14,8 +14,8 @@ import java.util.stream.Collectors; /** * Factory of scalar Java functions. * The purpose of this is to embellish anonymous functions with a runtime type - * such that they can be inspected and will return a parseable toString. - * + * such that they can be inspected and will return a parsable toString. + * * @author bratseth */ @Beta @@ -31,9 +31,9 @@ public class ScalarFunctions { public static DoubleUnaryOperator sqrt() { return new Sqrt(); } public static DoubleUnaryOperator square() { return new Square(); } - public static Function<List<Integer>, Double> random() { return new Random(); } - public static Function<List<Integer>, Double> equal(List<String> argumentNames) { return new EqualElements(argumentNames); } - public static Function<List<Integer>, Double> sum(List<String> argumentNames) { return new SumElements(argumentNames); } + public static Function<List<Long>, Double> random() { return new Random(); } + public static Function<List<Long>, Double> equal(List<String> argumentNames) { return new EqualElements(argumentNames); } + public static Function<List<Long>, Double> sum(List<String> argumentNames) { return new SumElements(argumentNames); } // Binary operators ----------------------------------------------------------------------------- @@ -60,7 +60,7 @@ public class ScalarFunctions { public static class Multiply implements DoubleBinaryOperator { @Override - public double applyAsDouble(double left, double right) { return left * right; } + public double applyAsDouble(double left, double right) { return left * right; } @Override public String toString() { return "f(a,b)(a * b)"; } } @@ -100,26 +100,26 @@ public class ScalarFunctions { // Variable-length operators ----------------------------------------------------------------------------- - public static class EqualElements implements Function<List<Integer>, Double> { - private final ImmutableList<String> argumentNames; + public static class EqualElements implements Function<List<Long>, Double> { + private final ImmutableList<String> argumentNames; private EqualElements(List<String> argumentNames) { this.argumentNames = ImmutableList.copyOf(argumentNames); } @Override - public Double apply(List<Integer> values) { + public Double apply(List<Long> values) { if (values.isEmpty()) return 1.0; - for (Integer value : values) + for (Long value : values) if ( ! value.equals(values.get(0))) return 0.0; return 1.0; } @Override - public String toString() { + public String toString() { if (argumentNames.size() == 0) return "1"; if (argumentNames.size() == 1) return "1"; if (argumentNames.size() == 2) return argumentNames.get(0) + "==" + argumentNames.get(1); - + StringBuilder b = new StringBuilder(); for (int i = 0; i < argumentNames.size() -1; i++) { b.append("(").append(argumentNames.get(i)).append("==").append(argumentNames.get(i+1)).append(")"); @@ -130,25 +130,25 @@ public class ScalarFunctions { } } - public static class Random implements Function<List<Integer>, Double> { + public static class Random implements Function<List<Long>, Double> { @Override - public Double apply(List<Integer> values) { + public Double apply(List<Long> values) { return ThreadLocalRandom.current().nextDouble(); } @Override public String toString() { return "random"; } } - public static class SumElements implements Function<List<Integer>, Double> { + public static class SumElements implements Function<List<Long>, Double> { private final ImmutableList<String> argumentNames; private SumElements(List<String> argumentNames) { this.argumentNames = ImmutableList.copyOf(argumentNames); } @Override - public Double apply(List<Integer> values) { - int sum = 0; - for (Integer value : values) + public Double apply(List<Long> values) { + long sum = 0; + for (Long value : values) sum += value; return (double)sum; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java index aabb53d1c67..1e830bac461 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java @@ -36,7 +36,7 @@ public class DenseBinaryFormat implements BinaryFormat { buffer.putInt1_4Bytes(tensor.type().dimensions().size()); for (int i = 0; i < tensor.type().dimensions().size(); i++) { buffer.putUtf8String(tensor.type().dimensions().get(i).name()); - buffer.putInt1_4Bytes(tensor.dimensionSizes().size(i)); + buffer.putInt1_4Bytes((int)tensor.dimensionSizes().size(i)); // XXX: Size truncation } } @@ -71,7 +71,7 @@ public class DenseBinaryFormat implements BinaryFormat { int dimensionCount = buffer.getInt1_4Bytes(); TensorType.Builder builder = new TensorType.Builder(); for (int i = 0; i < dimensionCount; i++) - builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes()); + builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes()); // XXX: Size truncation return builder.build(); } @@ -84,7 +84,7 @@ public class DenseBinaryFormat implements BinaryFormat { } private void decodeCells(DimensionSizes sizes, GrowableByteBuffer buffer, IndexedTensor.BoundBuilder builder) { - for (int i = 0; i < sizes.totalSize(); i++) + for (long i = 0; i < sizes.totalSize(); i++) builder.cellByDirectIndex(i, buffer.getDouble()); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java index 61dfa888567..34e6cccf0f0 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java @@ -46,16 +46,16 @@ class MixedBinaryFormat implements BinaryFormat { buffer.putInt1_4Bytes(denseDimensions.size()); for (TensorType.Dimension dimension : denseDimensions) { buffer.putUtf8String(dimension.name()); - buffer.putInt1_4Bytes(dimension.size().orElseThrow(() -> - new IllegalArgumentException("Unknown size of indexed dimension."))); + buffer.putInt1_4Bytes((int)dimension.size().orElseThrow(() -> + new IllegalArgumentException("Unknown size of indexed dimension.")).longValue()); // XXX: Size truncation } } private void encodeCells(GrowableByteBuffer buffer, MixedTensor tensor) { List<TensorType.Dimension> sparseDimensions = tensor.type().dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); - int denseSubspaceSize = tensor.denseSubspaceSize(); + long denseSubspaceSize = tensor.denseSubspaceSize(); if (sparseDimensions.size() > 0) { - buffer.putInt1_4Bytes(tensor.size() / denseSubspaceSize); + buffer.putInt1_4Bytes((int)(tensor.size() / denseSubspaceSize)); // XXX: Size truncation } Iterator<Tensor.Cell> cellIterator = tensor.cellIterator(); while (cellIterator.hasNext()) { @@ -98,7 +98,7 @@ class MixedBinaryFormat implements BinaryFormat { } int numIndexedDimensions = buffer.getInt1_4Bytes(); for (int i = 0; i < numIndexedDimensions; ++i) { - builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes()); + builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes()); // XXX: Size truncation } return builder.build(); } @@ -106,21 +106,21 @@ class MixedBinaryFormat implements BinaryFormat { private void decodeCells(GrowableByteBuffer buffer, MixedTensor.BoundBuilder builder, TensorType type) { List<TensorType.Dimension> sparseDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); TensorType sparseType = MixedTensor.createPartialType(sparseDimensions); - int denseSubspaceSize = builder.denseSubspaceSize(); + long denseSubspaceSize = builder.denseSubspaceSize(); int numBlocks = 1; if (sparseDimensions.size() > 0) { numBlocks = buffer.getInt1_4Bytes(); } - double[] denseSubspace = new double[denseSubspaceSize]; + double[] denseSubspace = new double[(int)denseSubspaceSize]; for (int i = 0; i < numBlocks; ++i) { TensorAddress.Builder sparseAddress = new TensorAddress.Builder(sparseType); for (TensorType.Dimension sparseDimension : sparseDimensions) { sparseAddress.add(sparseDimension.name(), buffer.getUtf8String()); } - for (int denseOffset = 0; denseOffset < denseSubspaceSize; denseOffset++) { - denseSubspace[denseOffset] = buffer.getDouble(); + for (long denseOffset = 0; denseOffset < denseSubspaceSize; denseOffset++) { + denseSubspace[(int)denseOffset] = buffer.getDouble(); } builder.block(sparseAddress.build(), denseSubspace); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java index 19969506eca..0cd3ff77aca 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java @@ -3,13 +3,14 @@ package com.yahoo.tensor.serialization; import com.google.common.annotations.Beta; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.tensor.MappedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; -import com.yahoo.text.Utf8; -import java.util.*; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Implementation of a sparse binary format for a tensor on the form: @@ -39,7 +40,7 @@ class SparseBinaryFormat implements BinaryFormat { } private void encodeCells(GrowableByteBuffer buffer, Tensor tensor) { - buffer.putInt1_4Bytes(tensor.size()); + buffer.putInt1_4Bytes((int)tensor.size()); // XXX: Size truncation for (Iterator<Tensor.Cell> i = tensor.cellIterator(); i.hasNext(); ) { Map.Entry<TensorAddress, Double> cell = i.next(); encodeAddress(buffer, cell.getKey()); @@ -79,8 +80,8 @@ class SparseBinaryFormat implements BinaryFormat { } private void decodeCells(GrowableByteBuffer buffer, Tensor.Builder builder, TensorType type) { - int numCells = buffer.getInt1_4Bytes(); - for (int i = 0; i < numCells; ++i) { + long numCells = buffer.getInt1_4Bytes(); // XXX: Size truncation + for (long i = 0; i < numCells; ++i) { Tensor.Builder.CellBuilder cellBuilder = builder.cell(); decodeAddress(buffer, cellBuilder, type); cellBuilder.value(buffer.getDouble()); diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 693b0f09351..38a8329bff1 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.tensor; import com.google.common.collect.ImmutableList; import com.yahoo.tensor.evaluation.MapEvaluationContext; import com.yahoo.tensor.evaluation.VariableTensor; -import com.yahoo.tensor.functions.Argmax; import com.yahoo.tensor.functions.ConstantTensor; import com.yahoo.tensor.functions.Join; import com.yahoo.tensor.functions.Reduce; @@ -12,14 +11,12 @@ import com.yahoo.tensor.functions.TensorFunction; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; import static com.yahoo.tensor.TensorType.Dimension.Type; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -99,7 +96,7 @@ public class TensorTestCase { ImmutableList.of("y", "x"))); assertEquals(Tensor.from("{ {x:0,y:0}:0, {x:0,y:1}:0, {x:1,y:0}:0, {x:1,y:1}:1, {x:2,y:0}:0, {x:2,y:1}:2, }"), Tensor.generate(new TensorType.Builder().indexed("x", 3).indexed("y", 2).build(), - (List<Integer> indexes) -> (double)indexes.get(0)*indexes.get(1))); + (List<Long> indexes) -> (double)indexes.get(0)*indexes.get(1))); assertEquals(Tensor.from("{ {x:0,y:0,z:0}:0, {x:0,y:1,z:0}:1, {x:1,y:0,z:0}:1, {x:1,y:1,z:0}:2, {x:2,y:0,z:0}:2, {x:2,y:1,z:0}:3, "+ " {x:0,y:0,z:1}:1, {x:0,y:1,z:1}:2, {x:1,y:0,z:1}:2, {x:1,y:1,z:1}:3, {x:2,y:0,z:1}:3, {x:2,y:1,z:1}:4 }"), Tensor.range(new TensorType.Builder().indexed("x", 3).indexed("y", 2).indexed("z", 2).build())); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h index 31185a9ff7c..023594d3018 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h @@ -35,7 +35,7 @@ public: size_t capacity() const { return _ht.capacity(); } size_t size() const { return _ht.size(); } bool empty() const { return _ht.empty(); } - insert_result insert(const value_type & value); + insert_result insert(const value_type & value) { return _ht.insert(value); } template <typename InputIt> void insert(InputIt first, InputIt last); const V & operator [] (const K & key) const { return _ht.find(key)->second; } diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp index 40ef90826b6..b526188b8b2 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp @@ -17,13 +17,7 @@ hash_map<K, V, H, EQ, M>::hash_map(size_t reserveSize, H hasher, EQ equality) : { } template <typename K, typename V, typename H, typename EQ, typename M> -hash_map<K, V, H, EQ, M>::~hash_map() { } - -template <typename K, typename V, typename H, typename EQ, typename M> -typename hash_map<K, V, H, EQ, M>::insert_result -hash_map<K, V, H, EQ, M>::insert(const value_type & value) { - return _ht.insert(value); -} +hash_map<K, V, H, EQ, M>::~hash_map() = default; template <typename K, typename V, typename H, typename EQ, typename M> void @@ -70,6 +64,8 @@ hash_map<K, V, H, EQ, M>::getMemoryUsed() const template class vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>; \ template vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insert_result \ vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insert(std::pair<K,V> &&); \ + template vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insert_result \ + vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insertInternal(std::pair<K,V> &&); \ template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>; #define VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, E) \ diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index 263ee952c2e..15949067a60 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -141,19 +141,18 @@ public: typedef Value* pointer; typedef std::forward_iterator_tag iterator_category; - iterator(hashtable * hash, next_t start) : _hash(start), _subNode(start), _hashTable(hash) { - advanceToNextValidHash(); - } - iterator(hashtable * hash, next_t start, next_t subNode) : _hash(start), _subNode(subNode), _hashTable(hash) { } - Value & operator * () const { return _hashTable->get(_subNode); } - Value * operator -> () const { return & _hashTable->get(_subNode); } - iterator & operator ++ () { - if (_hashTable->_nodes[_subNode].hasNext()) { - _subNode = _hashTable->_nodes[_subNode].getNext(); - } else { - _hash++; + iterator(hashtable * hash) : _current(0), _hashTable(hash) { + if ((_current < _hashTable->initializedSize()) && ! _hashTable->_nodes[_current].valid()) { advanceToNextValidHash(); } + } + iterator(hashtable * hash, next_t pos) : _current(pos), _hashTable(hash) { } + static iterator end(hashtable *hash) { return iterator(hash, Node::npos); } + + Value & operator * () const { return _hashTable->get(_current); } + Value * operator -> () const { return & _hashTable->get(_current); } + iterator & operator ++ () { + advanceToNextValidHash(); return *this; } iterator operator ++ (int) { @@ -161,19 +160,19 @@ public: ++(*this); return prev; } - bool operator==(const iterator& rhs) const { return (_subNode == rhs._subNode); } - bool operator!=(const iterator& rhs) const { return (_subNode != rhs._subNode); } + bool operator==(const iterator& rhs) const { return (_current == rhs._current); } + bool operator!=(const iterator& rhs) const { return (_current != rhs._current); } /// Carefull about this one. Only used by lrucache. - next_t getInternalIndex() const { return _subNode; } - void setInternalIndex(next_t n) { _subNode = n; } - next_t getHash() const { return _hash; } + next_t getInternalIndex() const { return _current; } + void setInternalIndex(next_t n) { _current = n; } private: void advanceToNextValidHash() { - for (;(_hash < _hashTable->getTableSize()) && ! _hashTable->_nodes[_hash].valid(); _hash++) { } - _subNode = (_hash < _hashTable->getTableSize()) ? _hash : Node::npos; + for (_current++;(_current < _hashTable->initializedSize()) && ! _hashTable->_nodes[_current].valid(); _current++) { } + if (_current >= _hashTable->initializedSize()) { + _current = Node::npos; + } } - next_t _hash; - next_t _subNode; + next_t _current; hashtable * _hashTable; friend class hashtable::const_iterator; @@ -186,21 +185,19 @@ public: typedef const Value* pointer; typedef std::forward_iterator_tag iterator_category; - const_iterator(const hashtable * hash, next_t start) : _hash(start), _subNode(start), _hashTable(hash) { - advanceToNextValidHash(); - } - const_iterator(const hashtable * hash, next_t start, next_t subNode) : _hash(start), _subNode(subNode), _hashTable(hash) { } - const_iterator(const iterator &i) - : _hash(i._hash), _subNode(i._subNode), _hashTable(i._hashTable) {} - const Value & operator * () const { return _hashTable->get(_subNode); } - const Value * operator -> () const { return & _hashTable->get(_subNode); } - const_iterator & operator ++ () { - if (_hashTable->_nodes[_subNode].hasNext()) { - _subNode = _hashTable->_nodes[_subNode].getNext(); - } else { - _hash++; + const_iterator(const hashtable * hash) : _current(0), _hashTable(hash) { + if ((_current < _hashTable->initializedSize()) && ! _hashTable->_nodes[_current].valid()) { advanceToNextValidHash(); } + } + const_iterator(const hashtable * hash, next_t pos) : _current(pos), _hashTable(hash) { } + const_iterator(const iterator &i) : _current(i._current), _hashTable(i._hashTable) {} + static const_iterator end(const hashtable *hash) { return const_iterator(hash, Node::npos); } + + const Value & operator * () const { return _hashTable->get(_current); } + const Value * operator -> () const { return & _hashTable->get(_current); } + const_iterator & operator ++ () { + advanceToNextValidHash(); return *this; } const_iterator operator ++ (int) { @@ -208,17 +205,17 @@ public: ++(*this); return prev; } - bool operator==(const const_iterator& rhs) const { return (_subNode == rhs._subNode); } - bool operator!=(const const_iterator& rhs) const { return (_subNode != rhs._subNode); } - next_t getInternalIndex() const { return _subNode; } - next_t getHash() const { return _hash; } + bool operator==(const const_iterator& rhs) const { return (_current == rhs._current); } + bool operator!=(const const_iterator& rhs) const { return (_current != rhs._current); } + next_t getInternalIndex() const { return _current; } private: void advanceToNextValidHash() { - for (;(_hash < _hashTable->getTableSize()) && ! _hashTable->_nodes[_hash].valid(); _hash++) { } - _subNode = (_hash < _hashTable->getTableSize()) ? _hash : Node::npos; + for (_current++;(_current < _hashTable->initializedSize()) && ! _hashTable->_nodes[_current].valid(); _current++) { } + if (_current >= _hashTable->initializedSize()) { + _current = Node::npos; + } } - next_t _hash; - next_t _subNode; + next_t _current; const hashtable * _hashTable; }; typedef std::pair<iterator, bool> insert_result; @@ -231,10 +228,10 @@ public: hashtable(size_t reservedSpace); hashtable(size_t reservedSpace, const Hash & hasher, const Equal & equal); virtual ~hashtable(); - iterator begin() { return iterator(this, 0); } - iterator end() { return iterator(this, Node::npos); } - const_iterator begin() const { return const_iterator(this, 0); } - const_iterator end() const { return const_iterator(this, Node::npos); } + iterator begin() { return iterator(this); } + iterator end() { return iterator::end(this); } + const_iterator begin() const { return const_iterator(this); } + const_iterator end() const { return const_iterator::end(this); } size_t capacity() const { return _nodes.capacity(); } size_t size() const { return _count; } bool empty() const { return _count == 0; } @@ -249,7 +246,9 @@ public: const_iterator find(const AltKey & key) const { return find<AltKey, AltExtract, AltHash, AltEqual>(key, AltExtract()); } const_iterator find(const Key & key) const; template <typename V> - insert_result insert(V && node); + insert_result insert(V && node) { + return insertInternal(std::forward<V>(node)); + } void erase(const Key & key); void reserve(size_t sz) { if (sz > _nodes.capacity()) { @@ -280,7 +279,8 @@ protected: Value & getByInternalIndex(size_t index) { return _nodes[index].getValue(); } const Value & getByInternalIndex(size_t index) const { return _nodes[index].getValue(); } template <typename MoveHandler> - void erase(MoveHandler & moveHandler, const const_iterator & key); + void erase(MoveHandler & moveHandler, next_t h, const const_iterator & key); + next_t hash(const Key & key) const { return modulator(_hasher(key)); } private: Modulator _modulator; size_t _count; @@ -292,7 +292,7 @@ private: const Value & get(size_t index) const { return _nodes[index].getValue(); } next_t modulator(next_t key) const { return _modulator.modulo(key); } next_t getTableSize() const { return _modulator.getTableSize(); } - next_t hash(const Key & key) const { return modulator(_hasher(key)); } + size_t initializedSize() const { return _nodes.size(); } template <typename MoveHandler> void move(MoveHandler & moveHandler, next_t from, next_t to) { _nodes[to] = std::move(_nodes[from]); diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp index 359e71aa0d2..f499ba35f3f 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.hpp @@ -67,11 +67,10 @@ typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::iterator hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) { next_t h = hash(key); - if (_nodes[h].valid()) { - next_t start(h); + if (__builtin_expect(_nodes[h].valid(), true)) { do { - if (_equal(_keyExtractor(_nodes[h].getValue()), key)) { - return iterator(this, start, h); + if (__builtin_expect(_equal(_keyExtractor(_nodes[h].getValue()), key), true)) { + return iterator(this, h); } h = _nodes[h].getNext(); } while (h != Node::npos); @@ -84,11 +83,10 @@ typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::const_iterat hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const Key & key) const { next_t h = hash(key); - if (_nodes[h].valid()) { - next_t start(h); + if (__builtin_expect(_nodes[h].valid(), true)) { do { - if (_equal(_keyExtractor(_nodes[h].getValue()), key)) { - return const_iterator(this, start, h); + if (__builtin_expect(_equal(_keyExtractor(_nodes[h].getValue()), key), true)) { + return const_iterator(this, h); } h = _nodes[h].getNext(); } while (h != Node::npos); @@ -104,11 +102,10 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & k AltHash altHasher; next_t h = modulator(altHasher(key)); if (_nodes[h].valid()) { - next_t start(h); AltEqual altEqual; do { if (altEqual(altExtract(_keyExtractor(_nodes[h].getValue())), key)) { - return const_iterator(this, start, h); + return const_iterator(this, h); } h = _nodes[h].getNext(); } while (h != Node::npos); @@ -124,11 +121,10 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & k AltHash altHasher; next_t h = modulator(altHasher(key)); if (_nodes[h].valid()) { - next_t start(h); AltEqual altEqual; do { if (altEqual(altExtract(_keyExtractor(_nodes[h].getValue())), key)) { - return iterator(this, start, h); + return iterator(this, h); } h = _nodes[h].getNext(); } while (h != Node::npos); @@ -137,19 +133,12 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::find(const AltKey & k } template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > -template<typename V> -typename hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insert_result -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insert(V && node) { - return insertInternal(std::forward<V>(node)); -} - -template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > void hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::erase(const Key & key) { const_iterator found(find(key)); if (found != end()) { DefaultMoveHandler moveHandler; - erase(moveHandler, found); + erase(moveHandler, hash(key), found); } } @@ -169,11 +158,11 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insertInternal(V && n if ( ! _nodes[h].valid() ) { _nodes[h] = std::forward<V>(node); _count++; - return insert_result(iterator(this, h, h), true); + return insert_result(iterator(this, h), true); } else if (_nodes.size() <= _nodes.capacity()) { for (next_t c(h); c != Node::npos; c = _nodes[c].getNext()) { if (_equal(_keyExtractor(_nodes[c].getValue()), _keyExtractor(node))) { - return insert_result(iterator(this, h, c), false); + return insert_result(iterator(this, c), false); } } if (_nodes.size() < _nodes.capacity()) { @@ -182,7 +171,7 @@ hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::insertInternal(V && n _nodes[h].setNext(newIdx); new (_nodes.push_back_fast()) Node(std::forward<V>(node), p); _count++; - return insert_result(iterator(this, h, newIdx), true); + return insert_result(iterator(this, newIdx), true); } else { resize(_nodes.capacity()*2); return insertInternal(std::forward<V>(node)); @@ -214,9 +203,8 @@ void hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::reclaim(MoveHand template< typename Key, typename Value, typename Hash, typename Equal, typename KeyExtract, typename Modulator > template <typename MoveHandler> void -hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::erase(MoveHandler & moveHandler, const const_iterator & it) +hashtable<Key, Value, Hash, Equal, KeyExtract, Modulator>::erase(MoveHandler & moveHandler, next_t h, const const_iterator & it) { - next_t h = it.getHash(); next_t prev = Node::npos; do { if (h == it.getInternalIndex()) { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 15257e11cbe..4c932969460 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.curator; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.vespa.curator.recipes.CuratorCounter; import com.yahoo.vespa.zookeeper.ZooKeeperServer; @@ -21,7 +22,6 @@ import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.retry.ExponentialBackoffRetry; -import java.io.Closeable; import java.time.Duration; import java.util.Arrays; import java.util.Collections; @@ -68,16 +68,26 @@ public class Curator implements AutoCloseable { this(createConnectionSpec(configserverConfig)); } - private static String createConnectionSpec(ConfigserverConfig config) { + static String createConnectionSpec(ConfigserverConfig config) { + String thisServer = HostName.getLocalhost(); + StringBuilder sb = new StringBuilder(); for (int i = 0; i < config.zookeeperserver().size(); i++) { ConfigserverConfig.Zookeeperserver server = config.zookeeperserver(i); - sb.append(server.hostname()); - sb.append(":"); - sb.append(server.port()); - if (i < config.zookeeperserver().size() - 1) { - sb.append(","); + + String spec = String.format("%s:%d", server.hostname(), server.port()); + + if (config.zookeeperLocalhostAffinity() && server.hostname().equals(thisServer)) { + // Only connect to localhost server if possible, to save network traffic + // and balance load. + return spec; } + + if (sb.length() > 0) { + sb.append(','); + } + + sb.append(spec); } return sb.toString(); } diff --git a/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/CuratorTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java index 36205bdaca3..1899dcfe7cd 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/CuratorTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; +package com.yahoo.vespa.curator; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.vespa.curator.Curator; +import com.yahoo.net.HostName; import org.apache.curator.test.TestingServer; import org.junit.After; import org.junit.Before; @@ -11,7 +11,6 @@ import org.junit.Test; import java.io.IOException; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; /** @@ -74,6 +73,23 @@ public class CuratorTest { } } + @Test + public void localhost_affinity() { + String localhostHostName = "myhost"; + int localhostPort = 123; + String localhostSpec = localhostHostName + ":" + localhostPort; + + ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); + builder.zookeeperserver(createZKBuilder(localhostHostName, localhostPort)); + builder.zookeeperserver(createZKBuilder("otherhost", 345)); + builder.zookeeperLocalhostAffinity(true); + ConfigserverConfig config = new ConfigserverConfig(builder); + + HostName.setHostNameForTestingOnly(localhostHostName); + + assertThat(Curator.createConnectionSpec(config), is(localhostSpec)); + } + private ConfigserverConfig createTestConfig() { ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); builder.zookeeperserver(createZKBuilder("localhost", port1)); |