diff options
Diffstat (limited to 'container-core')
35 files changed, 1068 insertions, 179 deletions
diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt index 341155457a8..6132c253c13 100644 --- a/container-core/CMakeLists.txt +++ b/container-core/CMakeLists.txt @@ -8,6 +8,7 @@ install_config_definition(src/main/resources/configdefinitions/identity.def cont install_config_definition(src/main/resources/configdefinitions/log-handler.def container.core.log-handler.def) install_config_definition(src/main/resources/configdefinitions/metrics-packets-handler.def container.jdisc.state.metrics-packets-handler.def) install_config_definition(src/main/resources/configdefinitions/metrics-presentation.def metrics.metrics-presentation.def) +install_config_definition(src/main/resources/configdefinitions/metrics-proxy-api.def container.handler.metrics.metrics-proxy-api.def) install_config_definition(src/main/resources/configdefinitions/mockservice.def container.handler.test.mockservice.def) install_config_definition(src/main/resources/configdefinitions/qr-searchers.def container.qr-searchers.def) install_config_definition(src/main/resources/configdefinitions/qr.def container.qr.def) diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 5dc9a863970..6d683c53984 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -11,6 +11,23 @@ ], "fields": [] }, + "com.yahoo.container.handler.ClustersStatus$Require": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.container.handler.ClustersStatus$Require[] values()", + "public static com.yahoo.container.handler.ClustersStatus$Require valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.container.handler.ClustersStatus$Require ONE", + "public static final enum com.yahoo.container.handler.ClustersStatus$Require ALL" + ] + }, "com.yahoo.container.handler.ClustersStatus": { "superClass": "com.yahoo.component.AbstractComponent", "interfaces": [], @@ -23,7 +40,8 @@ "public void setReceiveTrafficByDefault(boolean)", "public void setUp(java.lang.Object)", "public void setDown(java.lang.Object)", - "public boolean containerShouldReceiveTraffic()" + "public boolean containerShouldReceiveTraffic()", + "public boolean containerShouldReceiveTraffic(com.yahoo.container.handler.ClustersStatus$Require)" ], "fields": [] }, @@ -154,6 +172,7 @@ "public void <init>(com.yahoo.container.handler.ThreadpoolConfig)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxthreads(int)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxThreadExecutionTimeSeconds(int)", + "public com.yahoo.container.handler.ThreadpoolConfig$Builder softStartSeconds(double)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -191,7 +210,8 @@ "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.container.handler.ThreadpoolConfig$Builder)", "public int maxthreads()", - "public int maxThreadExecutionTimeSeconds()" + "public int maxThreadExecutionTimeSeconds()", + "public double softStartSeconds()" ], "fields": [ "public static final java.lang.String CONFIG_DEF_MD5", @@ -230,6 +250,7 @@ "public void <init>(com.yahoo.container.QrSearchersConfig)", "public void <init>(com.yahoo.container.handler.ClustersStatus)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus)", + "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus)", "public void setInRotation(java.lang.Boolean)", @@ -853,5 +874,55 @@ "public bridge synthetic java.lang.Object clone()" ], "fields": [] + }, + "ai.vespa.cloud.Environment": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static ai.vespa.cloud.Environment[] values()", + "public static ai.vespa.cloud.Environment valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum ai.vespa.cloud.Environment dev", + "public static final enum ai.vespa.cloud.Environment perf", + "public static final enum ai.vespa.cloud.Environment test", + "public static final enum ai.vespa.cloud.Environment staging", + "public static final enum ai.vespa.cloud.Environment prod" + ] + }, + "ai.vespa.cloud.SystemInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.cloud.config.ConfigserverConfig)", + "public void <init>(ai.vespa.cloud.Zone)", + "public ai.vespa.cloud.Zone zone()" + ], + "fields": [] + }, + "ai.vespa.cloud.Zone": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(ai.vespa.cloud.Environment, java.lang.String)", + "public ai.vespa.cloud.Environment environment()", + "public java.lang.String region()", + "public java.lang.String toString()", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public static ai.vespa.cloud.Zone from(java.lang.String)" + ], + "fields": [] } -} +}
\ No newline at end of file diff --git a/container-core/pom.xml b/container-core/pom.xml index f3861c92129..64e5ebb00d3 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -16,6 +16,16 @@ <packaging>container-plugin</packaging> <dependencies> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>http-utils</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </dependency> + + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> @@ -227,6 +237,16 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.github.tomakehurst</groupId> + <artifactId>wiremock-standalone</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/container-core/src/main/java/ai/vespa/cloud/Environment.java b/container-core/src/main/java/ai/vespa/cloud/Environment.java new file mode 100644 index 00000000000..8f1d9fc962a --- /dev/null +++ b/container-core/src/main/java/ai/vespa/cloud/Environment.java @@ -0,0 +1,13 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.cloud; + +/** + * The environments of a Vespa cloud instance + * + * @author bratseth + */ +public enum Environment { + + dev, perf, test, staging, prod + +} diff --git a/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java b/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java new file mode 100644 index 00000000000..0524ae072cd --- /dev/null +++ b/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.cloud; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; + +/** + * Provides information about the system in which this container is running. + * This is available and can be injected when running in a cloud environment. + * + * @author bratseth + */ +public class SystemInfo { + + private final Zone zone; + + /** Do not use */ + @Inject + public SystemInfo(ConfigserverConfig config) { + this.zone = new Zone(Environment.valueOf(config.environment()), config.region()); + } + + /** Create an instance for testing */ + public SystemInfo(Zone zone) { + this.zone = zone; + } + + /** Returns the zone this is running in */ + public Zone zone() { return zone; } + +} diff --git a/container-core/src/main/java/ai/vespa/cloud/Zone.java b/container-core/src/main/java/ai/vespa/cloud/Zone.java new file mode 100644 index 00000000000..48293aa7908 --- /dev/null +++ b/container-core/src/main/java/ai/vespa/cloud/Zone.java @@ -0,0 +1,61 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.cloud; + +import java.util.Objects; + +/** + * The zone in which a cloud deployment may be running. + * A zone is a combination of an environment and a region. + * + * @author bratseth + */ +public class Zone { + + private final Environment environment; + + private final String region; + + public Zone(Environment environment, String region) { + this.environment = environment; + this.region = region; + } + + public Environment environment() { return environment; } + public String region() { return region; } + + /** Returns the string environment.region */ + @Override + public String toString() { return environment + "." + region; } + + @Override + public int hashCode() { return Objects.hash(environment, region); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof Zone)) return false; + Zone other = (Zone)o; + return this.environment.equals(other.environment) && this.region.equals(other.region); + } + + /** + * Creates a zone from a string on the form environment.region + * + * @throws IllegalArgumentException if the given string is not a valid zone + */ + public static Zone from(String zoneString) { + String[] parts = zoneString.split("\\."); + if (parts.length != 2) + throw new IllegalArgumentException("A zone string must be on the form [environment].[region], but was '" + zoneString + "'"); + + Environment environment; + try { + environment = Environment.valueOf(parts[0]); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid zone '" + zoneString + "': No environment named '" + parts[0] + "'"); + } + return new Zone(environment, parts[1]); + } + +} diff --git a/container-core/src/main/java/ai/vespa/cloud/package-info.java b/container-core/src/main/java/ai/vespa/cloud/package-info.java new file mode 100644 index 00000000000..259a2bda258 --- /dev/null +++ b/container-core/src/main/java/ai/vespa/cloud/package-info.java @@ -0,0 +1,10 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Public API to the Vespa cloud, available when this container runs in a cloud. + */ +@ExportPackage +@PublicApi +package ai.vespa.cloud; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/Server.java b/container-core/src/main/java/com/yahoo/container/Server.java deleted file mode 100644 index a4dec6de5a2..00000000000 --- a/container-core/src/main/java/com/yahoo/container/Server.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container; - -import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.container.QrConfig.Rpc; - -/** - * The http server singleton managing listeners for various ports, - * and the threads used to respond to requests on the ports - * - * @author bratseth - * @deprecated - */ -@SuppressWarnings("deprecation") -@Deprecated // TODO: Remove this when the last usage og getServerDiscriminator is removed -public class Server { - - //TODO: Make this final again. - private static final Server instance = new Server(); - - /** A short string which is different for all the qrserver instances on a given node. */ - private String localServerDiscriminator = "qrserver.0"; - - private Server() { } - - public static Server get() { - return instance; - } - - public void initialize(QrConfig config) { - localServerDiscriminator = config.discriminator(); - } - - /** - * A string unique for this QRS on this server. - * - * @return a server specific string - * @deprecated do not use - */ - @Deprecated - public String getServerDiscriminator() { - return localServerDiscriminator; - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java index 4c3d76436dd..ca11ad387ee 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java @@ -148,10 +148,9 @@ public class BundleLoader { /** * Returns the bundles that are not assumed to be retained by the new application generation. - * and cleans up the map of active file references. Note that at this point we don't yet know - * the full set of new bundles, because of the potential pre-install directives in the new bundles. - * However, only "disk bundles" (file:) can be listed in the pre-install directive, so we know - * about all the obsolete application bundles. + * Note that at this point we don't yet know the full set of new bundles, because of the potential + * pre-install directives in the new bundles. However, only "disk bundles" (file:) can be listed + * in the pre-install directive, so we know about all the obsolete application bundles. */ private Set<Bundle> getObsoleteBundles(List<FileReference> newReferences) { Set<Bundle> bundlesToRemove = new HashSet<>(osgi.getCurrentBundles()); @@ -165,6 +164,9 @@ public class BundleLoader { return bundlesToRemove; } + /** + * Cleans up the map of active file references + */ private void removeInactiveFileReferences(List<FileReference> newReferences) { // Clean up the map of active bundles Set<FileReference> fileReferencesToRemove = getObsoleteFileReferences(newReferences); @@ -184,6 +186,7 @@ public class BundleLoader { // The bundle at index 0 for each file reference always corresponds to the bundle at the file reference location Set<Bundle> allowedDuplicates = obsoleteReferences.stream() + .filter(reference -> ! isDiskBundle(reference)) .map(reference -> reference2Bundles.get(reference).get(0)) .collect(Collectors.toSet()); diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java index aab13a1cc7b..0ed0daa2141 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java @@ -27,6 +27,8 @@ public class ClustersStatus extends AbstractComponent { @Inject public ClustersStatus() { } + public enum Require {ONE, ALL} + /** Are there any (in-service influencing) clusters in this container? */ private boolean containerHasClusters; @@ -72,12 +74,25 @@ public class ClustersStatus extends AbstractComponent { setDown((String) clusterIdentifier); } - /** Returns whether this container should receive traffic based on the state of this */ + @Deprecated // TODO: Remove on Vespa 8 public boolean containerShouldReceiveTraffic() { + return containerShouldReceiveTraffic(Require.ONE); + } + /** + * Returns whether this container should receive traffic based on the state of this + * @param require Requirement for being up, ALL or ONE. + */ + public boolean containerShouldReceiveTraffic(Require require) { synchronized (mutex) { if (containerHasClusters) { - // Should receive traffic when at least one cluster is up - return clusterStatus.values().stream().anyMatch(status -> status==true); + switch (require) { + case ONE: + // Should receive traffic when at least one cluster is up + return clusterStatus.values().stream().anyMatch(status -> status == true); + case ALL: + default: + return !clusterStatus.isEmpty() && clusterStatus.values().stream().allMatch(status -> status == true); + } } else { return true; diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index 4cc3b48fd1a..b427a58c9b7 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -9,6 +9,7 @@ import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.protect.ProcessTerminator; import com.yahoo.jdisc.Metric; +import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -43,7 +44,8 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex threadpoolConfig.maxthreads(), 0L, TimeUnit.SECONDS, new SynchronousQueue<>(false), - ThreadFactoryFactory.getThreadFactory("threadpool")); + ThreadFactoryFactory.getThreadFactory("threadpool"), + metric); // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also // get the dreaded thread locals initialized even if they will never run. // That counters what we we want to achieve with the Q that will prefer thread locality. @@ -161,17 +163,22 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex /** A thread pool executor which maintains the last time a worker completed */ private final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { + private static final String UNHANDLED_EXCEPTIONS_METRIC = "jdisc.thread_pool.unhandled_exceptions"; + volatile long lastThreadAssignmentTimeMillis = System.currentTimeMillis(); private final AtomicLong startedCount = new AtomicLong(0); private final AtomicLong completedCount = new AtomicLong(0); + private final Metric metric; public WorkerCompletionTimingThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, - ThreadFactory threadFactory) { + ThreadFactory threadFactory, + Metric metric) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); + this.metric = metric; } @Override @@ -185,6 +192,9 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex protected void afterExecute(Runnable r, Throwable t) { super.afterExecute(r, t); completedCount.incrementAndGet(); + if (t != null) { + metric.add(UNHANDLED_EXCEPTIONS_METRIC, 1L, metric.createContext(Map.of("exception", t.getClass().getSimpleName()))); + } } @Override diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index b9ef1627ce7..0bf86e8f440 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -24,6 +24,8 @@ public class VipStatus { /** If this is non-null, its value decides whether this container is in rotation */ private Boolean rotationOverride = null; + private final boolean initiallyInRotation; + /** The current state of this */ private boolean currentlyInRotation; @@ -44,20 +46,29 @@ public class VipStatus { this(new QrSearchersConfig.Builder().build(), clustersStatus); } + /** For testing */ public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus) { - this(dispatchers, clustersStatus, new StateMonitor()); + this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, new StateMonitor()); } @Inject - public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) { + public VipStatus(QrSearchersConfig dispatchers, + VipStatusConfig vipStatusConfig, + ClustersStatus clustersStatus, + StateMonitor healthState) { this.clustersStatus = clustersStatus; this.healthState = healthState; + initiallyInRotation = vipStatusConfig.initiallyInRotation(); healthState.status(StateMonitor.Status.initializing); clustersStatus.setContainerHasClusters(! dispatchers.searchcluster().isEmpty()); updateCurrentlyInRotation(); } - /** @deprecated don't pass VipStatusConfig */ + @Deprecated // TODO: Remove on Vespa 8 + public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) { + this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, healthState); + } + @Deprecated // TODO: Remove on Vespa 8 public VipStatus(QrSearchersConfig dispatchers, VipStatusConfig ignored, ClustersStatus clustersStatus) { this(dispatchers, clustersStatus); @@ -102,10 +113,20 @@ public class VipStatus { private void updateCurrentlyInRotation() { synchronized (mutex) { - if (rotationOverride != null) + if (rotationOverride != null) { currentlyInRotation = rotationOverride; - else - currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(); + } else { + if (healthState.status() == StateMonitor.Status.up) { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ONE); + } + else if (healthState.status() == StateMonitor.Status.initializing) { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL) + && initiallyInRotation; + } + else { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL); + } + } // Change to/from 'up' when appropriate but don't change 'initializing' to 'down' if (currentlyInRotation) diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java new file mode 100644 index 00000000000..321f7b3994a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java @@ -0,0 +1,32 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.Map; +import java.util.logging.Logger; + +import static java.util.logging.Level.WARNING; + +/** + * @author gjoranv + */ +public class ErrorResponse extends JsonResponse { + private static Logger log = Logger.getLogger(ErrorResponse.class.getName()); + + private static ObjectMapper objectMapper = new ObjectMapper(); + + public ErrorResponse(int code, String message) { + super(code, asErrorJson(message)); + } + + static String asErrorJson(String message) { + try { + return objectMapper.writeValueAsString(Map.of("error", message)); + } catch (JsonProcessingException e) { + log.log(WARNING, "Could not encode error message to json:", e); + return "Could not encode error message to json, check the log for details."; + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java new file mode 100644 index 00000000000..92840cee48f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java @@ -0,0 +1,78 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.restapi.Path; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.Response.Status.OK; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static java.util.logging.Level.WARNING; + +/** + * @author gjoranv + */ +public abstract class HttpHandlerBase extends ThreadedHttpRequestHandler { + + protected HttpHandlerBase(Executor executor) { + super(executor); + } + + protected abstract Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer); + + @Override + public final HttpResponse handle(HttpRequest request) { + if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported"); + + Path path = new Path(request.getUri()); + + return doHandle(request.getUri(), path, getConsumer(request)) + .orElse(new ErrorResponse(NOT_FOUND, "No content at given path")); + } + + private String getConsumer(HttpRequest request) { + return request.getProperty("consumer"); + } + + protected JsonResponse resourceListResponse(URI requestUri, List<String> resources) { + try { + return new JsonResponse(OK, resourceList(requestUri, resources)); + } catch (JSONException e) { + log.log(WARNING, "Bad JSON construction in generated resource list for " + requestUri.getPath(), e); + return new ErrorResponse(INTERNAL_SERVER_ERROR, + "An error occurred when generating the list of api resources."); + } + } + + // TODO: Use jackson with a "Resources" class instead of JSONObject + private static String resourceList(URI requestUri, List<String> resources) throws JSONException { + int port = requestUri.getPort(); + String host = requestUri.getHost(); + StringBuilder base = new StringBuilder("http://"); + base.append(host); + if (port >= 0) { + base.append(":").append(port); + } + String uriBase = base.toString(); + JSONArray linkList = new JSONArray(); + for (String api : resources) { + JSONObject resource = new JSONObject(); + resource.put("url", uriBase + api); + linkList.put(resource); + } + return new JSONObject().put("resources", linkList).toString(4); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java new file mode 100644 index 00000000000..def06ce9de3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java @@ -0,0 +1,30 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; + +/** + * @author gjoranv + */ +public class JsonResponse extends HttpResponse { + private final byte[] data; + + public JsonResponse(int code, String data) { + super(code); + this.data = data.getBytes(Charset.forName(DEFAULT_CHARACTER_ENCODING)); + } + + @Override + public String getContentType() { + return "application/json"; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(data); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java new file mode 100644 index 00000000000..78ea62e1b3a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java @@ -0,0 +1,77 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import ai.vespa.util.http.VespaHttpClientBuilder; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; +import com.yahoo.yolean.Exceptions; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicResponseHandler; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.OK; + +/** + * @author gjoranv + */ +public class MetricsV2Handler extends HttpHandlerBase { + + public static final String V2_PATH = "/metrics/v2"; + static final String VALUES_PATH = V2_PATH + "/values"; + + private static final int HTTP_CONNECT_TIMEOUT = 5000; + private static final int HTTP_SOCKET_TIMEOUT = 30000; + + private final String metricsProxyUri; + private final HttpClient httpClient = createHttpClient(); + + @Inject + public MetricsV2Handler(Executor executor, + MetricsProxyApiConfig config) { + super(executor); + metricsProxyUri = "http://localhost:" + config.metricsPort() + config.metricsApiPath(); + } + + @Override + protected Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private JsonResponse valuesResponse(String consumer) { + try { + String uri = metricsProxyUri + consumerQuery(consumer); + String metricsJson = httpClient.execute(new HttpGet(uri), new BasicResponseHandler()); + return new JsonResponse(OK, metricsJson); + } catch (IOException e) { + log.warning("Unable to retrieve metrics from " + metricsProxyUri + ": " + Exceptions.toMessageString(e)); + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + private static CloseableHttpClient createHttpClient() { + return VespaHttpClientBuilder.create() + .setUserAgent("application-metrics-retriever") + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout(HTTP_CONNECT_TIMEOUT) + .setSocketTimeout(HTTP_SOCKET_TIMEOUT) + .build()) + .build(); + } + + static String consumerQuery(String consumer) { + return (consumer == null || consumer.isEmpty()) ? "" : "?consumer=" + consumer; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/package-info.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/package-info.java new file mode 100644 index 00000000000..031886afc94 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/package-info.java @@ -0,0 +1,6 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** Exported config package */ +@ExportPackage +package com.yahoo.container.handler.metrics; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 705092ef12e..62dc81aa103 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -240,37 +240,43 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { renderStartTime, commitStartTime, endTime, - jdiscRequest.getUri().toString(), + getUri(jdiscRequest), extendedResponse.getParsedQuery(), extendedResponse.getTiming()); Optional<AccessLogEntry> jdiscRequestAccessLogEntry = AccessLoggingRequestHandler.getAccessLogEntry(jdiscRequest); - + AccessLogEntry entry; if (jdiscRequestAccessLogEntry.isPresent()) { - // This means we are running with Jetty, not Netty. + // The request is created by JDisc http layer (Jetty) // Actual logging will be done by the Jetty integration; here, we just need to populate. - httpResponse.populateAccessLogEntry(jdiscRequestAccessLogEntry.get()); - return; + entry = jdiscRequestAccessLogEntry.get(); + } else { + // Not running on JDisc http layer (Jetty), e.g unit tests + AccessLogEntry accessLogEntry = new AccessLogEntry(); + populateAccessLogEntryNotCreatedByHttpServer( + accessLogEntry, + jdiscRequest, + extendedResponse.getTiming(), + httpRequest.getUri().toString(), + commitStartTime, + startTime, + rendererWiring.written(), + httpResponse.getStatus()); + accessLog.log(accessLogEntry); + entry = accessLogEntry; } + httpResponse.populateAccessLogEntry(entry); + } - // We are running without Jetty. No access logging will be done at container level, so we do it here. - // TODO: Remove when netty support is removed. - - AccessLogEntry accessLogEntry = new AccessLogEntry(); - - populateAccessLogEntryNotCreatedByHttpServer( - accessLogEntry, - jdiscRequest, - extendedResponse.getTiming(), - httpRequest.getUri().toString(), - commitStartTime, - startTime, - rendererWiring.written(), - httpResponse.getStatus()); - httpResponse.populateAccessLogEntry(accessLogEntry); - - accessLog.log(accessLogEntry); + private String getUri(com.yahoo.jdisc.http.HttpRequest jdiscRequest) { + URI uri = jdiscRequest.getUri(); + StringBuilder builder = new StringBuilder(uri.getPath()); + String query = uri.getQuery(); + if (query != null && !query.isBlank()) { + builder.append('?').append(query); + } + return builder.toString(); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index dddde1205ca..c58d49bf8c8 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -13,6 +13,7 @@ import com.yahoo.log.LogLevel; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -31,6 +32,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler public static final String CONTENT_TYPE = "Content-Type"; private static final String RENDERING_ERRORS = "rendering_errors"; + private static final String UNHANDLED_EXCEPTIONS_METRIC = "jdisc.http.handler.unhandled_exceptions"; /** Logger for subclasses */ protected final Logger log; @@ -79,6 +81,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler channel.setHttpResponse(httpResponse); // may or may not have already been done render(httpRequest, httpResponse, channel, jdiscRequest.creationTime(TimeUnit.MILLISECONDS)); } catch (Exception e) { + metric.add(UNHANDLED_EXCEPTIONS_METRIC, 1L, contextFor(request, Map.of("exception", e.getClass().getSimpleName()))); metric.add(RENDERING_ERRORS, 1, null); log.log(LogLevel.ERROR, "Uncaught exception handling request", e); if (channel != null) { @@ -93,7 +96,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler /** Render and return whether the channel was closed */ private void render(HttpRequest request, HttpResponse httpResponse, - LazyContentChannel channel, long startTime) throws IOException { + LazyContentChannel channel, long startTime) { LoggingCompletionHandler logOnCompletion = null; ContentChannelOutputStream output = null; try { @@ -168,7 +171,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler @Override public void close(CompletionHandler completionHandler) { if ( closed ) return; - try { httpRequest.getData().close(); } catch (IOException e) {}; + try { httpRequest.getData().close(); } catch (IOException e) {} if (channel == null) channel = handleResponse(); try { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index 7789cae50aa..99732af9d31 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -2,7 +2,6 @@ package com.yahoo.container.jdisc; import com.google.inject.Inject; -import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.ResourceReference; @@ -18,14 +17,15 @@ import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.log.LogLevel; +import java.net.URI; import java.time.Duration; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import static java.util.Collections.singletonMap; /** * A request handler which assigns a worker thread to handle each request. @@ -79,21 +79,30 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { this.allowAsyncResponse = allowAsyncResponse; } - private Map<String, Metric.Context> handlerContexts = new CopyOnWriteHashMap<>(); - private Metric.Context contextFor(BindingMatch match) { + Metric.Context contextFor(Request request, Map<String, String> extraDimensions) { + BindingMatch match = request.getBindingMatch(); if (match == null) return null; UriPattern matched = match.matched(); if (matched == null) return null; String name = matched.toString(); - Metric.Context context = handlerContexts.get(name); - if (context == null) { - Map<String, String> dimensions = singletonMap("handler", name); - context = this.metric.createContext(dimensions); - handlerContexts.put(name, context); + String endpoint = request.headers().containsKey("Host") ? request.headers().get("Host").get(0) : null; + + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("handler", name); + if (endpoint != null) { + dimensions.put("endpoint", endpoint); } - return context; + URI uri = request.getUri(); + dimensions.put("scheme", uri.getScheme()); + dimensions.put("port", Integer.toString(uri.getPort())); + String handlerClassName = getClass().getName(); + dimensions.put("handler-name", handlerClassName); + dimensions.putAll(extraDimensions); + return this.metric.createContext(dimensions); } + private Metric.Context contextFor(Request request) { return contextFor(request, Map.of()); } + /** * Handles a request by assigning a worker thread to it. * @@ -101,7 +110,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { */ @Override public final ContentChannel handleRequest(Request request, ResponseHandler responseHandler) { - metric.add("handled.requests", 1, contextFor(request.getBindingMatch())); + metric.add("handled.requests", 1, contextFor(request)); if (request.getTimeout(TimeUnit.SECONDS) == null) { Duration timeout = getTimeout(); if (timeout != null) { @@ -195,7 +204,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { if ( tryHasResponded()) throw new IllegalStateException("Response already handled"); ContentChannel cc = responseHandler.handleResponse(response); long millis = request.timeElapsed(TimeUnit.MILLISECONDS); - metric.set("handled.latency", millis, contextFor(request.getBindingMatch())); + metric.set("handled.latency", millis, contextFor(request)); return cc; } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index f690c240537..faa08402cdc 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -14,6 +14,7 @@ import java.util.TreeSet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** @@ -27,7 +28,7 @@ public class StateMonitor extends AbstractComponent { private final static Logger log = Logger.getLogger(StateMonitor.class.getName()); - public enum Status {up, down, initializing}; + public enum Status {up, down, initializing} private final CopyOnWriteArrayList<StateMetricConsumer> consumers = new CopyOnWriteArrayList<>(); private final Thread thread; @@ -37,6 +38,7 @@ public class StateMonitor extends AbstractComponent { private volatile MetricSnapshot snapshot; private volatile Status status; private final TreeSet<String> valueNames = new TreeSet<>(); + private final AtomicBoolean stopped = new AtomicBoolean(false); /** For testing */ public StateMonitor() { @@ -53,10 +55,16 @@ public class StateMonitor extends AbstractComponent { } StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) { + this((long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)), + Status.valueOf(config.initialStatus()), + timer, threadFactory); + } + /* For Testing */ + public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory) { this.timer = timer; - this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)); + this.snapshotIntervalMs = snapshotIntervalMS; this.lastSnapshotTimeMs = timer.currentTimeMillis(); - this.status = Status.valueOf(config.initialStatus()); + this.status = status; thread = threadFactory.newThread(this::run); thread.start(); } @@ -99,13 +107,13 @@ public class StateMonitor extends AbstractComponent { private void run() { log.finest("StateMonitor started."); try { - while (!Thread.interrupted()) { - checkTime(); - Thread.sleep((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis()); + synchronized (stopped) { + while (!stopped.get()) { + checkTime(); + stopped.wait((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis()); + } } - } catch (InterruptedException e) { - - } + } catch (InterruptedException e) { } log.finest("StateMonitor stopped."); } @@ -137,12 +145,13 @@ public class StateMonitor extends AbstractComponent { @Override public void deconstruct() { - thread.interrupt(); + synchronized (stopped) { + stopped.set(true); + stopped.notifyAll(); + } try { thread.join(5000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + } catch (InterruptedException e) { } if (thread.isAlive()) { log.warning("StateMonitor failed to terminate within 5 seconds of interrupt signal. Ignoring."); } diff --git a/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java b/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java index 17ed321331b..32ea3ae708f 100644 --- a/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java @@ -1,33 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; - /** * A 200 ok response with a message in JSON. * * @author bratseth */ -public class MessageResponse extends HttpResponse { - - private final Slime slime = new Slime(); +public class MessageResponse extends SlimeJsonResponse { public MessageResponse(String message) { - super(200); - slime.setObject().setString("message", message); + super(slime(message)); } - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); + private static Slime slime(String message) { + var slime = new Slime(); + slime.setObject().setString("message", message); + return slime; } - @Override - public String getContentType() { return "application/json"; } - } diff --git a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java index ff301d44798..0188136addb 100644 --- a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java @@ -1,46 +1,41 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.net.URI; +import java.util.List; /** * Returns a response containing an array of links to sub-resources * * @author bratseth */ -public class ResourceResponse extends HttpResponse { +public class ResourceResponse extends SlimeJsonResponse { - private final Slime slime = new Slime(); + public ResourceResponse(URI parentUrl, List<String> subResources) { + super(200, toSlime(parentUrl, subResources)); + } public ResourceResponse(URI parentUrl, String ... subResources) { - super(200); - Cursor resourceArray = slime.setObject().setArray("resources"); - for (String subResource : subResources) { - Cursor resourceEntry = resourceArray.addObject(); - resourceEntry.setString("url", new Uri(parentUrl).append(subResource) - .withTrailingSlash() - .toString()); - } + this(parentUrl, List.of(subResources)); } public ResourceResponse(HttpRequest request, String ... subResources) { this(request.getUri(), subResources); } - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); + private static Slime toSlime(URI parentUrl, List<String> subResources) { + var slime = new Slime(); + var resourceArray = slime.setObject().setArray("resources"); + for (var subResource : subResources) { + var resourceEntry = resourceArray.addObject(); + resourceEntry.setString("url", new Uri(parentUrl).append(subResource) + .withTrailingSlash() + .toString()); + } + return slime; } - @Override - public String getContentType() { return "application/json"; } - } diff --git a/container-core/src/main/resources/configdefinitions/container-http.def b/container-core/src/main/resources/configdefinitions/container-http.def index ccf559b862a..23edd402893 100644 --- a/container-core/src/main/resources/configdefinitions/container-http.def +++ b/container-core/src/main/resources/configdefinitions/container-http.def @@ -3,3 +3,6 @@ namespace=container.core ## If non-empty, handlers should emit a header containing this string as key and the local host name as value hostResponseHeaderKey string default="" + +## For debugging, number of requests to add trace and timing information too if debugging is enabled. +numQueriesToTraceOnDebugAfterConstruction int default=1000 diff --git a/container-core/src/main/resources/configdefinitions/health-monitor.def b/container-core/src/main/resources/configdefinitions/health-monitor.def index 5e70c72ae3f..4e91d85b2b8 100644 --- a/container-core/src/main/resources/configdefinitions/health-monitor.def +++ b/container-core/src/main/resources/configdefinitions/health-monitor.def @@ -6,4 +6,4 @@ namespace=container.jdisc.config snapshot_interval double default=300 # Initial status used in /state/v1/health API (value for 'code' in 'status'). See StateMonitor for valid values -initialStatus string default="up" +initialStatus string default="initializing" diff --git a/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def new file mode 100644 index 00000000000..3e5b973e3f3 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def @@ -0,0 +1,6 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace=container.handler.metrics + +metricsPort int +metricsApiPath string diff --git a/container-core/src/main/resources/configdefinitions/threadpool.def b/container-core/src/main/resources/configdefinitions/threadpool.def index 5b5e7e2f4a2..9bb9badd9b5 100644 --- a/container-core/src/main/resources/configdefinitions/threadpool.def +++ b/container-core/src/main/resources/configdefinitions/threadpool.def @@ -8,3 +8,8 @@ maxthreads int default=500 # get out of a bad state. This should be set a bit higher than the expected max execution # time of each request when in a state of overload, i.e about "worst case execution time*2" maxThreadExecutionTimeSeconds int default=190 + +# Length of period for soft start +# During this period number of availble threads will be gradually increased. +# Currently used to avoid feeding overload in container during cold start. +softStartSeconds double default=0 diff --git a/container-core/src/main/sh/vespa-load-balancer-status b/container-core/src/main/sh/vespa-load-balancer-status index d7383c1614c..e9cadc487a1 100755 --- a/container-core/src/main/sh/vespa-load-balancer-status +++ b/container-core/src/main/sh/vespa-load-balancer-status @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. diff --git a/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java b/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java new file mode 100644 index 00000000000..6bc8b395e00 --- /dev/null +++ b/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.cloud; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class SystemInfoTest { + + @Test + public void testSystemInfo() { + Zone zone = new Zone(Environment.dev, "us-west-1"); + SystemInfo info = new SystemInfo(zone); + assertEquals(zone, info.zone()); + } + + @Test + public void testZone() { + Zone zone = Zone.from("dev.us-west-1"); + zone = Zone.from(zone.toString()); + assertEquals(Environment.dev, zone.environment()); + assertEquals("us-west-1", zone.region()); + Zone sameZone = Zone.from("dev.us-west-1"); + assertEquals(sameZone.hashCode(), zone.hashCode()); + assertEquals(sameZone, zone); + + try { + Zone.from("invalid"); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("A zone string must be on the form [environment].[region], but was 'invalid'", + e.getMessage()); + } + + try { + Zone.from("invalid.us-west-1"); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Invalid zone 'invalid.us-west-1': No environment named 'invalid'", e.getMessage()); + } + } + +} diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index 52679c15957..e13debcddda 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -4,6 +4,9 @@ package com.yahoo.container.handler; import static org.junit.Assert.*; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.container.jdisc.state.StateMonitor; +import com.yahoo.jdisc.core.SystemTimer; import org.junit.Test; /** @@ -13,44 +16,101 @@ import org.junit.Test; */ public class VipStatusTestCase { - @Test - public void testVipStatusWorksWithClusters() { + private static QrSearchersConfig getSearchersConfig(String[] clusters) { var b = new QrSearchersConfig.Builder(); - var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); - searchClusterB.name("cluster1"); - searchClusterB.name("cluster2"); - searchClusterB.name("cluster3"); - b.searchcluster(searchClusterB); - VipStatus v = new VipStatus(b.build()); + if (clusters.length > 0) { + var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); + for (String cluster : clusters) { + searchClusterB.name(cluster); + } + b.searchcluster(searchClusterB); + } + return b.build(); + } + + private static VipStatus getVipStatus(String[] clusters, StateMonitor.Status startState, boolean initiallyInRotation) { + return new VipStatus(getSearchersConfig(clusters), + new VipStatusConfig.Builder().initiallyInRotation(initiallyInRotation).build(), + new ClustersStatus(), + new StateMonitor(1000, startState, new SystemTimer(), runnable -> { + Thread thread = new Thread(runnable, "StateMonitor"); + thread.setDaemon(true); + return thread; + })); + } - String cluster1 = "cluster1"; - String cluster2 = "cluster2"; - String cluster3 = "cluster3"; + private static void remove(String[] clusters, VipStatus v) { + for (String s : clusters) { + v.removeFromRotation(s); + } + } + + private static void add(String[] clusters, VipStatus v) { + for (String s : clusters) { + v.addToRotation(s); + } + } + private static void verifyUpOrDown(String[] clusters, StateMonitor.Status status) { + VipStatus v = getVipStatus(clusters, status, true); + remove(clusters, v); // initial state assertFalse(v.isInRotation()); + v.addToRotation(clusters[0]); + assertFalse(v.isInRotation()); + v.addToRotation(clusters[1]); + assertFalse(v.isInRotation()); + v.addToRotation(clusters[2]); + assertTrue(v.isInRotation()); + } + + @Test + public void testInitializingOrDownRequireAllUp() { + String[] clusters = {"cluster1", "cluster2", "cluster3"}; + verifyUpOrDown(clusters, StateMonitor.Status.initializing); + verifyUpOrDown(clusters, StateMonitor.Status.down); + } + + @Test + public void testUpRequireAllDown() { + String[] clusters = {"cluster1", "cluster2", "cluster3"}; - // one cluster becomes up - v.addToRotation(cluster1); + VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, true); + assertFalse(v.isInRotation()); + add(clusters, v); assertTrue(v.isInRotation()); - // all clusters down - v.removeFromRotation(cluster1); - v.removeFromRotation(cluster2); - v.removeFromRotation(cluster3); + v.removeFromRotation(clusters[0]); + assertTrue(v.isInRotation()); + v.removeFromRotation(clusters[1]); + assertTrue(v.isInRotation()); + v.removeFromRotation(clusters[2]); + assertFalse(v.isInRotation()); // All down + v.addToRotation(clusters[1]); assertFalse(v.isInRotation()); - // some clusters down - v.addToRotation(cluster2); + v.addToRotation(clusters[0]); + v.addToRotation(clusters[2]); + assertTrue(v.isInRotation()); // All up + v.removeFromRotation(clusters[0]); + v.removeFromRotation(clusters[2]); assertTrue(v.isInRotation()); - // all clusters up - v.addToRotation(cluster1); - v.addToRotation(cluster3); + v.addToRotation(clusters[0]); + v.addToRotation(clusters[2]); assertTrue(v.isInRotation()); - // and down again - v.removeFromRotation(cluster1); - v.removeFromRotation(cluster2); - v.removeFromRotation(cluster3); + } + + @Test + public void testNoClustersConfiguringInitiallyInRotationFalse() { + String[] clusters = {}; + VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, false); assertFalse(v.isInRotation()); } -} + @Test + public void testNoClustersConfiguringInitiallyInRotationTrue() { + String[] clusters = {}; + VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, true); + assertTrue(v.isInRotation()); + } + +}
\ No newline at end of file diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/ErrorResponseTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/ErrorResponseTest.java new file mode 100644 index 00000000000..882f9044dce --- /dev/null +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/ErrorResponseTest.java @@ -0,0 +1,25 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author gjoranv + */ +public class ErrorResponseTest { + + @Test + public void error_message_is_wrapped_in_json_object() { + var json = ErrorResponse.asErrorJson("bad"); + assertEquals("{\"error\":\"bad\"}", json); + } + + @Test + public void quotes_are_escaped() { + var json = ErrorResponse.asErrorJson("Message \" with \" embedded quotes."); + assertEquals("{\"error\":\"Message \\\" with \\\" embedded quotes.\"}", json); + } + +} diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java new file mode 100644 index 00000000000..b57814e50aa --- /dev/null +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java @@ -0,0 +1,143 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static com.yahoo.container.handler.metrics.MetricsV2Handler.V2_PATH; +import static com.yahoo.container.handler.metrics.MetricsV2Handler.VALUES_PATH; +import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + */ +public class MetricsV2HandlerTest { + + private static final String URI_BASE = "http://localhost"; + + private static final String V2_URI = URI_BASE + V2_PATH; + private static final String VALUES_URI = URI_BASE + VALUES_PATH; + + // Mock applicationmetrics api + private static final String MOCK_METRICS_PATH = "/node0"; + + private static final String TEST_FILE = "application-metrics.json"; + private static final String RESPONSE = getFileContents(TEST_FILE); + private static final String CPU_METRIC = "cpu.util"; + private static final String REPLACED_CPU_METRIC = "replaced_cpu_util"; + private static final String CUSTOM_CONSUMER = "custom-consumer"; + + private static RequestHandlerTestDriver testDriver; + + @Rule + public WireMockRule wireMockRule = new WireMockRule(options().dynamicPort()); + + @Before + public void setup() { + setupWireMock(); + var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(), + new MetricsProxyApiConfig.Builder() + .metricsPort(wireMockRule.port()) + .metricsApiPath(MOCK_METRICS_PATH) + .build()); + testDriver = new RequestHandlerTestDriver(handler); + } + + private void setupWireMock() { + wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) + .willReturn(aResponse().withBody(RESPONSE))); + + // Add a slightly different response for a custom consumer. + String myConsumerResponse = RESPONSE.replaceAll(CPU_METRIC, REPLACED_CPU_METRIC); + wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) + .withQueryParam("consumer", equalTo(CUSTOM_CONSUMER)) + .willReturn(aResponse().withBody(myConsumerResponse))); + } + + @Test + public void v2_response_contains_values_uri() throws Exception { + String response = testDriver.sendRequest(V2_URI).readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("resources")); + + JSONArray resources = root.getJSONArray("resources"); + assertEquals(1, resources.length()); + + JSONObject valuesUri = resources.getJSONObject(0); + assertEquals(VALUES_URI, valuesUri.getString("url")); + } + + @Ignore + @Test + public void visually_inspect_values_response() throws Exception { + JSONObject responseJson = getResponseAsJson(null); + System.out.println(responseJson.toString(4)); + } + + @Test + public void invalid_path_yields_error_response() throws Exception { + String response = testDriver.sendRequest(V2_URI + "/invalid").readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("error")); + assertTrue(root.getString("error" ).startsWith("No content")); + } + + @Test + public void values_response_is_equal_to_test_file() { + String response = testDriver.sendRequest(VALUES_URI).readAll(); + assertEquals(RESPONSE, response); + } + + @Test + public void consumer_is_propagated_to_metrics_proxy_api() throws JSONException { + JSONObject responseJson = getResponseAsJson(CUSTOM_CONSUMER); + + JSONObject firstNodeMetricsValues = + responseJson.getJSONArray("nodes").getJSONObject(0) + .getJSONObject("node") + .getJSONArray("metrics").getJSONObject(0) + .getJSONObject("values"); + + assertTrue(firstNodeMetricsValues.has(REPLACED_CPU_METRIC)); + } + + private JSONObject getResponseAsJson(String consumer) { + String response = testDriver.sendRequest(VALUES_URI + consumerQuery(consumer)).readAll(); + try { + return new JSONObject(response); + } catch (JSONException e) { + fail("Failed to create json object: " + e.getMessage()); + throw new RuntimeException(e); + } + } + + private static String getFileContents(String filename) { + InputStream in = MetricsV2HandlerTest.class.getClassLoader().getResourceAsStream(filename); + if (in == null) { + throw new RuntimeException("File not found: " + filename); + } + return new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining("\n")); + } + +} diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandlerTest.java new file mode 100644 index 00000000000..07dba21e5b6 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandlerTest.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.jdisc.Metric; +import org.junit.Test; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author bjorncs + */ +public class ThreadedHttpRequestHandlerTest { + + @Test + public void unhandled_exceptions_metric_is_incremented_if_subclassed_handler_throws_exception() { + MetricMock metricMock = new MetricMock(); + ThreadedHttpRequestHandlerThrowingException handler = new ThreadedHttpRequestHandlerThrowingException(metricMock); + RequestHandlerTestDriver driver = new RequestHandlerTestDriver(handler); + + driver.sendRequest("http://localhost/myhandler"); + String expectedMetricName = "jdisc.http.handler.unhandled_exceptions"; + assertThat(metricMock.addInvocations) + .containsKey(expectedMetricName); + assertThat(metricMock.addInvocations.get(expectedMetricName).dimensions) + .containsEntry("exception", "DummyException"); + } + + private static class MetricMock implements Metric { + final ConcurrentHashMap<String, SimpleMetricContext> addInvocations = new ConcurrentHashMap<>(); + + @Override public void add(String key, Number val, Context ctx) { + addInvocations.put(key, (SimpleMetricContext)ctx); + } + @Override public void set(String key, Number val, Context ctx) {} + @Override public Context createContext(Map<String, ?> properties) { return new SimpleMetricContext(properties); } + } + + private static class SimpleMetricContext implements Metric.Context { + final Map<String, String> dimensions; + + @SuppressWarnings("unchecked") + SimpleMetricContext(Map<String, ?> dimensions) { this.dimensions = (Map<String, String>)dimensions; } + } + + private static class ThreadedHttpRequestHandlerThrowingException extends ThreadedHttpRequestHandler { + ThreadedHttpRequestHandlerThrowingException(Metric metric) { + super(Executors.newSingleThreadExecutor(), metric); + } + @Override public HttpResponse handle(HttpRequest request) { throw new DummyException(); } + } + + private static class DummyException extends RuntimeException {} +}
\ No newline at end of file diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java index a806df1883f..8a1640e2c0e 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java @@ -1,3 +1,4 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; import com.fasterxml.jackson.databind.JsonNode; @@ -56,7 +57,8 @@ public class StateHandlerTestBase { HealthMonitorConfig healthMonitorConfig = new HealthMonitorConfig( new HealthMonitorConfig.Builder() - .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL))); + .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL)) + .initialStatus("up")); ThreadFactory threadFactory = ignored -> mock(Thread.class); this.monitor = new StateMonitor(healthMonitorConfig, timer, threadFactory); builder.guiceModules().install(new AbstractModule() { diff --git a/container-core/src/test/resources/application-metrics.json b/container-core/src/test/resources/application-metrics.json new file mode 100644 index 00000000000..52cbb721bb1 --- /dev/null +++ b/container-core/src/test/resources/application-metrics.json @@ -0,0 +1,92 @@ +{ + "nodes": [ + { + "hostname": "node0", + "role": "role0", + "node": { + "timestamp": 1234, + "metrics": [ + { + "values": { + "cpu.util": 16.222 + }, + "dimensions": { + "state": "active" + } + } + ] + }, + "services": [ + { + "name": "searchnode", + "timestamp": 1234, + "status": { + "code": "up" + }, + "metrics": [ + { + "values": { + "queries.count": 4 + }, + "dimensions": { + "documentType": "music" + } + } + ] + } + ] + }, + { + "hostname": "node1", + "role": "role1", + "node": { + "timestamp": 1234, + "metrics": [ + { + "values": { + "cpu.util": 32.444 + }, + "dimensions": { + "state": "active" + } + } + ] + }, + "services": [ + { + "name": "searchnode", + "timestamp": 1234, + "status": { + "code": "up" + }, + "metrics": [ + { + "values": { + "queries.count": 8 + }, + "dimensions": { + "documentType": "music" + } + } + ] + }, + { + "name": "slobrok", + "timestamp": 1234, + "status": { + "code": "unknown", + "description": "Unable to fetch metrics from service 'slobrok'" + }, + "metrics": [ + { + "values": {}, + "dimensions": { + "instance": "slobrok0" + } + } + ] + } + ] + } + ] +} |