diff options
Diffstat (limited to 'container-core/src/main')
131 files changed, 9018 insertions, 0 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/ConfigHack.java b/container-core/src/main/java/com/yahoo/container/ConfigHack.java new file mode 100644 index 00000000000..49965d0621d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/ConfigHack.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container; + +import com.yahoo.container.config.StatisticsEmitter; + +/** + * Distribution point for QRS specific stuff in a more or less + * container agnostic way. This is only a stepping stone to moving these things + * to Container and other pertinent classes, or simply removing the problems. + * + * <p> + * This class should not reach a final release. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class ConfigHack { + private volatile StatisticsEmitter statisticsEmitter = new StatisticsEmitter(); + public static final String TILED_TEMPLATE = "tiled"; + + public static final ConfigHack instance = new ConfigHack(); + + public StatisticsEmitter getStatisticsHandler() { + return statisticsEmitter; + } + + public void setStatisticsEmitter(StatisticsEmitter statisticsEmitter) { + this.statisticsEmitter = statisticsEmitter; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java new file mode 100755 index 00000000000..92b68e8a1ae --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -0,0 +1,183 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.FileReference; +import com.yahoo.container.core.config.BundleLoader; +import com.yahoo.container.osgi.AbstractRpcAdaptor; +import com.yahoo.container.osgi.ContainerRpcAdaptor; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.filedistribution.fileacquirer.FileAcquirerFactory; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.service.ClientProvider; +import com.yahoo.jdisc.service.ServerProvider; +import com.yahoo.osgi.Osgi; +import com.yahoo.vespa.config.ConfigTransformer; +import com.yahoo.vespa.config.ConfigTransformer.PathAcquirer; + +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * The container instance. This is a Vespa internal object, external code should + * only depend on this if there are no other options, and must be prepared to + * see it change at no warning. + * + * @author bratseth + */ +public class Container { + + private volatile boolean usingCustomFileAcquirer = false; + + private volatile ComponentRegistry<RequestHandler> requestHandlerRegistry; + private volatile ComponentRegistry<ClientProvider> clientProviderRegistry; + private volatile ComponentRegistry<ServerProvider> serverProviderRegistry; + private volatile ComponentRegistry<AbstractComponent> componentRegistry; + private volatile FileAcquirer fileAcquirer; + private Osgi osgi; + + private final ContainerRpcAdaptor rpcAdaptor = new ContainerRpcAdaptor(osgi); + + private volatile BundleLoader bundleLoader; + + private static Logger logger = Logger.getLogger(Container.class.getName()); + + //TODO: Make this final again. + private static Container instance = new Container(); + + public static Container get() { return instance; } + + public void setOsgi(Osgi osgi) { + this.osgi = osgi; + bundleLoader = new BundleLoader(osgi); + } + + public void shutdown() { + com.yahoo.container.Server.get().shutdown(); + if (fileAcquirer != null) + fileAcquirer.shutdown(); + + rpcAdaptor.shutdown(); + } + + /** Returns the rpc adaptor owned by this */ + public ContainerRpcAdaptor getRpcAdaptor() { + return rpcAdaptor; + } + + //Used to acquire files originating from the application package. + public FileAcquirer getFileAcquirer() { + return fileAcquirer; + } + + public BundleLoader getBundleLoader() { + if (bundleLoader == null) + bundleLoader = new BundleLoader(null); + return bundleLoader; + } + + /** Hack. For internal use only, will be removed later + * + * Used by LocalApplication to be able to repeatedly set up containers. + **/ + public static void resetInstance() { + instance = new Container(); + com.yahoo.container.Server.resetInstance(); + } + + /** + * Add an application specific RPC adaptor. + * + * @param adaptor the RPC adaptor to add to the Container + */ + public void addOptionalRpcAdaptor(AbstractRpcAdaptor adaptor) { + rpcAdaptor.bindRpcAdaptor(adaptor); + } + + public ComponentRegistry<RequestHandler> getRequestHandlerRegistry() { + return requestHandlerRegistry; + } + + public void setRequestHandlerRegistry(ComponentRegistry<RequestHandler> requestHandlerRegistry) { + this.requestHandlerRegistry = requestHandlerRegistry; + } + + public ComponentRegistry<ClientProvider> getClientProviderRegistry() { + return clientProviderRegistry; + } + + public void setClientProviderRegistry(ComponentRegistry<ClientProvider> clientProviderRegistry) { + this.clientProviderRegistry = clientProviderRegistry; + } + + public ComponentRegistry<ServerProvider> getServerProviderRegistry() { + return serverProviderRegistry; + } + + public void setServerProviderRegistry(ComponentRegistry<ServerProvider> serverProviderRegistry) { + this.serverProviderRegistry = serverProviderRegistry; + } + + public ComponentRegistry<AbstractComponent> getComponentRegistry() { + return componentRegistry; + } + + public void setComponentRegistry(ComponentRegistry<AbstractComponent> registry) { + registry.freeze(); + this.componentRegistry = registry; + } + + //Only intended for use by the Server instance. + public void setupFileAcquirer(QrConfig.Filedistributor filedistributorConfig) { + if (usingCustomFileAcquirer) + return; + + if (filedistributorConfig.configid().isEmpty()) { + if (fileAcquirer != null) + logger.warning("Disabling file distribution"); + fileAcquirer = null; + } else { + fileAcquirer = FileAcquirerFactory.create(filedistributorConfig.configid()); + } + + setPathAcquirer(fileAcquirer); + } + + /** Just a helper method to return a useful host to bind to. */ + public static String bindHostName(String host) { + if ("".equals(host)) { + return "0.0.0.0"; + } else { + return host; + } + } + + /** + * Only for internal use. + */ + public void setCustomFileAcquirer(final FileAcquirer fileAcquirer) { + if (this.fileAcquirer != null) { + throw new RuntimeException("Can't change file acquirer. Is " + + this.fileAcquirer + " attempted to set to " + fileAcquirer); + } + usingCustomFileAcquirer = true; + this.fileAcquirer = fileAcquirer; + setPathAcquirer(fileAcquirer); + } + + private void setPathAcquirer(final FileAcquirer fileAcquirer) { + ConfigTransformer.setPathAcquirer(new PathAcquirer() { + @Override + public Path getPath(FileReference fileReference) { + try { + return fileAcquirer.waitFor(fileReference, 15, TimeUnit.MINUTES).toPath(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + }); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/Server.java b/container-core/src/main/java/com/yahoo/container/Server.java new file mode 100644 index 00000000000..a68477ff668 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/Server.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container; + +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.container.QrConfig.Rpc; +import com.yahoo.container.osgi.ContainerRpcAdaptor; + +/** + * The http server singleton managing listeners for various ports, + * and the threads used to respond to requests on the ports + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class Server { + + //TODO: Make this final again. + private static Server instance = new Server(); + private ConfigSubscriber subscriber = new ConfigSubscriber(); + + /** The OSGi container instance of this server */ + private Container container=Container.get(); + + /** A short string which is different for all the qrserver instances on a given node. */ + private String localServerDiscriminator = "qrserver.0"; + + /** Creates a new server instance. Not usually useful, use get() to get the current server */ + private Server() { } + + /** The number of currently active incoming search requests */ + public int searchQueriesInFlight() { + //TODO: Implement + return 0; + } + + /** + * An estimate of current number of connections. It is better to be + * inaccurate than to acquire a lock per query fsync. + * + * @return The current number of open search connections + */ + public int getCurrentConnections() { + //TODO: Implement + return 0; + } + + public static Server get() { + return instance; + } + + + private void initRpcServer(Rpc rpcConfig) { + if (rpcConfig.enabled()) { + ContainerRpcAdaptor rpcAdaptor = container.getRpcAdaptor(); + rpcAdaptor.listen(rpcConfig.port()); + rpcAdaptor.setSlobrokId(rpcConfig.slobrokId()); + } + } + + /** Ugly hack, see Container.resetInstance + **/ + static void resetInstance() { + instance = new Server(); + } + + // TODO: Don't throw exception + // TODO: Make independent of config + public void initialize(QrConfig config) throws Exception { + //TODO: Reenable + //mBeanServer = new MBeanServer(configId); + + localServerDiscriminator = config.discriminator(); + container.setupFileAcquirer(config.filedistributor()); + initRpcServer(config.rpc()); + } + + /** + * A string unique for this QRS on this server. + * + * @return a server specific string + */ + public String getServerDiscriminator() { + return localServerDiscriminator; + } + + public void shutdown() { + if (subscriber!=null) subscriber.close(); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/config/StatisticsEmitter.java b/container-core/src/main/java/com/yahoo/container/config/StatisticsEmitter.java new file mode 100644 index 00000000000..abab07b044b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/config/StatisticsEmitter.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.config; + + +import com.yahoo.container.jdisc.HttpRequest; + +/** + * An insulating layer between HTTP and whatever kind of HTTP statistics + * interface is made available. This is an intermediary step towards a + * generalized network layer. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class StatisticsEmitter { + + public StringBuilder respond(HttpRequest request) { + return new StringBuilder("No statistics available yet."); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java b/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java new file mode 100644 index 00000000000..6c36e83a21e --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.config; + +import com.google.inject.Inject; +import com.yahoo.container.ConfigHack; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.concurrent.Executor; + +/** + * Handler of statistics http requests. Temporary hack as a step towards a more + * general network interface. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class StatisticsRequestHandler extends ThreadedHttpRequestHandler { + + @Inject + public StatisticsRequestHandler(Executor executor) { + super(executor); + } + + @Override + public HttpResponse handle(HttpRequest request) { + return new StatisticsResponse(ConfigHack.instance.getStatisticsHandler().respond(request)); + } + + protected static class StatisticsResponse extends HttpResponse { + + private final StringBuilder string; + + private StatisticsResponse(StringBuilder stringBuilder) { + super(com.yahoo.jdisc.http.HttpResponse.Status.OK); + this.string = stringBuilder; + } + + @Override + public void render(OutputStream stream) throws IOException { + Writer osWriter = new OutputStreamWriter(stream, Utf8.getCharset()); + osWriter.write(string.toString()); + osWriter.flush(); + osWriter.close(); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/config/package-info.java b/container-core/src/main/java/com/yahoo/container/config/package-info.java new file mode 100644 index 00000000000..e117e7127d2 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/config/testutil/TestUtil.java b/container-core/src/main/java/com/yahoo/container/config/testutil/TestUtil.java new file mode 100644 index 00000000000..3f6b5931e3e --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/config/testutil/TestUtil.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.config.testutil; + +import com.yahoo.container.core.config.HandlersConfigurerDi; + +import java.io.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author gjoranv + */ +public class TestUtil { + + public static void createComponentsConfig(String configFile, + String componentsFile, + String componentType) throws IOException { + createComponentsConfig(configFile, componentsFile, componentType, false); + } + + /** + * Copies the component ids from another config, e.g. 'handlers' to a 'components' array in a new components file, + * to avoid a manually written 'components' file for tests where the bundle spec is given by the component id. + * @param configFile Full path to the original config file, e.g. 'handlers' + * @param componentsFile Full path to the new 'components' file + * @param componentType The type of component, e.g. 'handler' + * @param append 'true' will append to an already existing 'componentsFile' + */ + public static void createComponentsConfig(String configFile, + String componentsFile, + String componentType, + boolean append) throws IOException { + StringBuilder buf = new StringBuilder(); + String line; + int i = 0; + if (append) { + final Pattern p = Pattern.compile("^[a-z]+" + "\\[\\d+\\]\\.id (.+)"); + BufferedReader reader = new BufferedReader(new InputStreamReader( + new FileInputStream(new File(componentsFile)), "UTF-8")); + while ((line = reader.readLine()) != null) { + Matcher m = p.matcher(line); + if (m.matches() && !m.group(1).equals(HandlersConfigurerDi.RegistriesHack.class.getName())) { + buf.append("components[").append(i).append("].id ").append(m.group(1)).append("\n"); + i++; + } + } + reader.close(); + } + BufferedReader reader = new BufferedReader(new InputStreamReader( + new FileInputStream(new File(configFile)), "UTF-8")); + final Pattern component = Pattern.compile("^" + componentType + "\\[\\d+\\]\\.id (.+)"); + while ((line = reader.readLine()) != null) { + Matcher m = component.matcher(line); + if (m.matches()) { + buf.append("components[").append(i).append("].id ").append(m.group(1)).append("\n"); + i++; + } + } + buf.append("components[").append(i).append("].id "). + append(HandlersConfigurerDi.RegistriesHack.class.getName()).append("\n"); + i++; + reader.close(); + buf.insert(0, "components["+i+"]\n"); + + Writer writer = new OutputStreamWriter(new FileOutputStream(new File(componentsFile)), "UTF-8"); + writer.write(buf.toString()); + writer.flush(); + writer.close(); + } + + /** + * Copies src file to dst file. If the dst file does not exist, it is created. + */ + public static void copyFile(String srcName, File dstFile) throws IOException { + InputStream src = new FileInputStream(new File(srcName)); + OutputStream dst = new FileOutputStream(dstFile); + byte[] buf = new byte[1024]; + int len; + while ((len = src.read(buf)) > 0) { + dst.write(buf, 0, len); + } + src.close(); + dst.close(); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/config/webapp/.gitignore b/container-core/src/main/java/com/yahoo/container/config/webapp/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/config/webapp/.gitignore diff --git a/container-core/src/main/java/com/yahoo/container/core/BundleLoaderProperties.java b/container-core/src/main/java/com/yahoo/container/core/BundleLoaderProperties.java new file mode 100644 index 00000000000..d9ff342c107 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/BundleLoaderProperties.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.core; + +/** + * @author gjoranv + * @since 5.46 + */ +public interface BundleLoaderProperties { + // TODO: This should be removed. The prefix is used to separate the bundles in BundlesConfig + // into those that are transferred with filedistribution and those that are preinstalled + // on disk. Instead, the model should have put them in two different configs. I.e. create a new + // config 'preinstalled-bundles.def'. + public static final String DISK_BUNDLE_PREFIX = "file:"; +} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java new file mode 100644 index 00000000000..682abcc53ac --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.core.config; + +import com.yahoo.collections.PredicateSplit; +import com.yahoo.config.FileReference; +import com.yahoo.container.Container; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.osgi.Osgi; +import org.osgi.framework.Bundle; +import org.osgi.framework.wiring.BundleRevision; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import static com.yahoo.collections.PredicateSplit.partition; +import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX; + +/** + * Manages the set of installed 3rd-party component bundles. + * + * @author tonytv + */ +public class BundleLoader { + + private final List<Bundle> initialBundles; + + private final Map<FileReference, List<Bundle>> reference2Bundles = new LinkedHashMap<>(); + + private final Logger log = Logger.getLogger(BundleLoader.class.getName()); + private final Osgi osgi; + + public BundleLoader(Osgi osgi) { + this.osgi = osgi; + initialBundles = Arrays.asList(osgi.getBundles()); + } + + private List<Bundle> obtainBundles(FileReference reference, FileAcquirer fileAcquirer) + throws InterruptedException { + File file = fileAcquirer.waitFor(reference, 15, TimeUnit.MINUTES); + return osgi.install(file.getAbsolutePath()); + } + + /** + * @return the number of bundles installed by this call. + */ + private int install(List<FileReference> references) { + Set<FileReference> bundlesToInstall = new HashSet<>(references); + bundlesToInstall.removeAll(reference2Bundles.keySet()); + + PredicateSplit<FileReference> bundlesToInstall_isDisk = partition(bundlesToInstall, BundleLoader::isDiskBundle); + installBundlesFromDisk(bundlesToInstall_isDisk.trueValues); + installBundlesFromFileDistribution(bundlesToInstall_isDisk.falseValues); + + startBundles(); + return bundlesToInstall.size(); + } + + private static boolean isDiskBundle(FileReference fileReference) { + return fileReference.value().startsWith(DISK_BUNDLE_PREFIX); + } + + private void installBundlesFromDisk(List<FileReference> bundlesToInstall) { + for (FileReference reference : bundlesToInstall) { + try { + installBundleFromDisk(reference); + } + catch(Exception e) { + throw new RuntimeException("Could not install bundle '" + reference + "'", e); + } + } + } + + private void installBundlesFromFileDistribution(List<FileReference> bundlesToInstall) { + if (!bundlesToInstall.isEmpty()) { + FileAcquirer fileAcquirer = Container.get().getFileAcquirer(); + boolean hasFileDistribution = (fileAcquirer != null); + if (hasFileDistribution) { + installWithFileDistribution(bundlesToInstall, fileAcquirer); + } else { + log.warning("Can't retrieve bundles since file distribution is disabled."); + } + } + } + + private void installBundleFromDisk(FileReference reference) { + assert(reference.value().startsWith(DISK_BUNDLE_PREFIX)); + String referenceFileName = reference.value().substring(DISK_BUNDLE_PREFIX.length()); + log.info("Installing bundle from disk with reference '" + reference.value() + "'"); + + File file = new File(referenceFileName); + if (!file.exists()) { + throw new IllegalArgumentException("Reference '" + reference.value() + "' not found on disk."); + } + + List<Bundle> bundles = osgi.install(file.getAbsolutePath()); + reference2Bundles.put(reference, bundles); + } + + private void installWithFileDistribution(List<FileReference> bundlesToInstall, FileAcquirer fileAcquirer) { + for (FileReference reference : bundlesToInstall) { + try { + log.info("Installing bundle with reference '" + reference.value() + "'"); + List<Bundle> bundles = obtainBundles(reference, fileAcquirer); + reference2Bundles.put(reference, bundles); + } + catch(Exception e) { + throw new RuntimeException("Could not install bundle '" + reference + "'", e); + } + } + } + + //all bundles must have been started first to ensure correct package resolution. + private void startBundles() { + for (List<Bundle> bundles : reference2Bundles.values()) { + for (Bundle bundle : bundles) { + try { + if (!isFragment(bundle)) + bundle.start(); + } catch(Exception e) { + throw new RuntimeException("Could not start bundle '" + bundle.getSymbolicName() + "'", e); + } + } + } + } + + // The OSGi APIs are just getting worse... + private boolean isFragment(Bundle bundle) { + BundleRevision bundleRevision = bundle.adapt(BundleRevision.class); + if (bundleRevision == null) + throw new NullPointerException("Null bundle revision means that bundle has probably been uninstalled: " + + bundle.getSymbolicName() + ":" + bundle.getVersion()); + return (bundleRevision.getTypes() & BundleRevision.TYPE_FRAGMENT) != 0; + } + + /** + * Returns the number of uninstalled bundles + */ + private int retainOnly(List<FileReference> newReferences) { + Set<Bundle> bundlesToRemove = new HashSet<>(Arrays.asList(osgi.getBundles())); + + for (FileReference fileReferenceToKeep: newReferences) { + if (reference2Bundles.containsKey(fileReferenceToKeep)) + bundlesToRemove.removeAll(reference2Bundles.get(fileReferenceToKeep)); + } + + bundlesToRemove.removeAll(initialBundles); + for (Bundle bundle : bundlesToRemove) { + log.info("Removing bundle '" + bundle.toString() + "'"); + osgi.uninstall(bundle); + } + + Set<FileReference> fileReferencesToRemove = new HashSet<>(reference2Bundles.keySet()); + fileReferencesToRemove.removeAll(newReferences); + + for (FileReference fileReferenceToRemove : fileReferencesToRemove) { + reference2Bundles.remove(fileReferenceToRemove); + } + return bundlesToRemove.size(); + } + + public synchronized int use(List<FileReference> bundles) { + int removedBundles = retainOnly(bundles); + int installedBundles = install(bundles); + startBundles(); + + log.info(removedBundles + " bundles were removed, and " + installedBundles + " bundles were installed."); + log.info(installedBundlesMessage()); + return removedBundles + installedBundles; + } + + private String installedBundlesMessage() { + StringBuilder sb = new StringBuilder("Installed bundles: {" ); + for (Bundle b : osgi.getBundles()) + sb.append("[" + b.getBundleId() + "]" + b.getSymbolicName() + ", "); + sb.setLength(sb.length() - 2); + sb.append("}"); + return sb.toString(); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java new file mode 100644 index 00000000000..0f56934c5bc --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.core.config; + +import com.google.inject.AbstractModule; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.FileReference; +import com.yahoo.container.core.DiagnosticsConfig; +import com.yahoo.container.di.ComponentDeconstructor; +import com.yahoo.container.di.Container; +import com.yahoo.container.di.componentgraph.core.ComponentGraph; +import com.yahoo.container.di.componentgraph.core.DotGraph; +import com.yahoo.container.di.config.SubscriberFactory; +import com.yahoo.container.di.osgi.OsgiUtil; +import com.yahoo.container.handler.observability.OverviewHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.container.logging.AccessLogInterface; +import com.yahoo.container.protect.FreezeDetector; +import com.yahoo.jdisc.application.OsgiFramework; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.service.ClientProvider; +import com.yahoo.jdisc.service.ServerProvider; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.log.LogLevel; +import com.yahoo.osgi.OsgiImpl; +import com.yahoo.statistics.Statistics; + +import org.osgi.framework.Bundle; +import org.osgi.framework.wiring.BundleWiring; +import scala.collection.immutable.Set; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.logging.Logger; + +import static com.yahoo.collections.CollectionUtil.first; +import static com.yahoo.container.util.Util.quote; + + +/** + * For internal use only. + * + * @author tonytv + * @author gjoranv + */ +//TODO: rename +public class HandlersConfigurerDi { + + private static final Logger log = Logger.getLogger(HandlersConfigurerDi.class.getName()); + + public static class RegistriesHack { + + @Inject + public RegistriesHack(com.yahoo.container.Container vespaContainer, + ComponentRegistry<AbstractComponent> allComponents, + ComponentRegistry<RequestHandler> requestHandlerRegistry, + ComponentRegistry<ClientProvider> clientProviderRegistry, + ComponentRegistry<ServerProvider> serverProviderRegistry) { + log.log(LogLevel.DEBUG, "RegistriesHack.init " + System.identityHashCode(this)); + + vespaContainer.setComponentRegistry(allComponents); + vespaContainer.setRequestHandlerRegistry(requestHandlerRegistry); + vespaContainer.setClientProviderRegistry(clientProviderRegistry); + vespaContainer.setServerProviderRegistry(serverProviderRegistry); + } + + } + + private final com.yahoo.container.Container vespaContainer; + private final OsgiWrapper osgiWrapper; + private final Container container; + + private volatile ComponentGraph currentGraph = new ComponentGraph(0); + + public HandlersConfigurerDi(SubscriberFactory subscriberFactory, + com.yahoo.container.Container vespaContainer, + String configId, + ComponentDeconstructor deconstructor, + Injector discInjector, + OsgiFramework osgiFramework) { + + this.vespaContainer = vespaContainer; + osgiWrapper = new OsgiWrapper(osgiFramework, vespaContainer.getBundleLoader()); + + container = new Container( + subscriberFactory, + configId, + deconstructor, + osgiWrapper + ); + try { + runOnceAndEnsureRegistryHackRun(discInjector); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted while setting up handlers for the first time."); + } + } + + private static class OsgiWrapper extends OsgiImpl implements com.yahoo.container.di.Osgi { + + private final OsgiFramework osgiFramework; + private final BundleLoader bundleLoader; + + public OsgiWrapper(OsgiFramework osgiFramework, BundleLoader bundleLoader) { + super(osgiFramework); + this.osgiFramework = osgiFramework; + this.bundleLoader = bundleLoader; + } + + + @Override + public BundleClasses getBundleClasses(ComponentSpecification bundleSpec, Set<String> packagesToScan) { + //Not written in an OO way since FelixFramework resides in JDisc core which for now is pure java, + //and to load from classpath one needs classes from scalalib. + + //Temporary hack: Using class name since ClassLoaderOsgiFramework is not available at compile time in this bundle. + if (osgiFramework.getClass().getName().equals("com.yahoo.application.container.impl.ClassLoaderOsgiFramework")) { + Bundle syntheticClassPathBundle = first(osgiFramework.bundles()); + ClassLoader classLoader = syntheticClassPathBundle.adapt(BundleWiring.class).getClassLoader(); + + return new BundleClasses( + syntheticClassPathBundle, + OsgiUtil.getClassEntriesForBundleUsingProjectClassPathMappings(classLoader, bundleSpec, packagesToScan)); + } else { + Bundle bundle = getBundle(bundleSpec); + if (bundle == null) + throw new RuntimeException("No bundle matching " + quote(bundleSpec)); + + return new BundleClasses(bundle, OsgiUtil.getClassEntriesInBundleClassPath(bundle, packagesToScan)); + } + } + + @Override + public void useBundles(Collection<FileReference> bundles) { + log.info("Installing bundles from the latest application"); + + List<FileReference> fileReferences = new ArrayList<>(bundles); + int bundlesRemovedOrInstalled = bundleLoader.use(fileReferences); + + if (bundlesRemovedOrInstalled > 0) { + refreshPackages(); + } + } + } + + public void runOnceAndEnsureRegistryHackRun(Injector discInjector) throws InterruptedException { + currentGraph = container.runOnce(currentGraph, createFallbackInjector(vespaContainer, discInjector)); + + RegistriesHack registriesHack = currentGraph.getInstance(RegistriesHack.class); + assert (registriesHack != null); + injectDotGraph(); + } + + @SuppressWarnings("deprecation") + private Injector createFallbackInjector(final com.yahoo.container.Container vespaContainer, Injector discInjector) { + return discInjector.createChildInjector(new AbstractModule() { + @Override + protected void configure() { + bind(com.yahoo.container.Container.class).toInstance(vespaContainer); + bind(com.yahoo.statistics.Statistics.class).toInstance(Statistics.nullImplementation); + bind(Linguistics.class).toInstance(new SimpleLinguistics()); + bind(FreezeDetector.class).toInstance(new FreezeDetector(new DiagnosticsConfig(new DiagnosticsConfig.Builder().disabled(true)))); + bind(AccessLog.class).toInstance(new AccessLog(new ComponentRegistry<>())); + bind(Executor.class).toInstance(Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("HandlersConfigurerDI"))); + + if (vespaContainer.getFileAcquirer() != null) + bind(com.yahoo.filedistribution.fileacquirer.FileAcquirer.class).toInstance(vespaContainer.getFileAcquirer()); + } + }); + } + + private void injectDotGraph() { + try { + OverviewHandler overviewHandler = currentGraph.getInstance(OverviewHandler.class); + overviewHandler.setDotGraph(DotGraph.generate(currentGraph)); + } catch (Exception e) { + log.fine("No overview handler"); + } + + } + + public void reloadConfig(long generation) { + container.reloadConfig(generation); + } + + public <T> T getComponent(Class<T> componentClass) { + return currentGraph.getInstance(componentClass); + } + + public void shutdown(ComponentDeconstructor deconstructor) { + container.shutdown(currentGraph, deconstructor); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java new file mode 100644 index 00000000000..b01b800a462 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -0,0 +1,132 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.core.config.testutil; + +import com.google.inject.Guice; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.container.Container; +import com.yahoo.container.di.CloudSubscriberFactory; +import com.yahoo.container.di.ComponentDeconstructor; +import com.yahoo.container.core.config.HandlersConfigurerDi; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.osgi.MockOsgi; + +import java.io.File; +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.Random; +import java.util.Set; + +/** + * Class for testing HandlersConfigurer. + * Not for public use. + * + * If possible, please avoid using this class and HandlersConfigurer in your tests + * @author tonytv + * @author gjoranv + * +*/ +public class HandlersConfigurerTestWrapper { + private ConfigSourceSet configSources = + new ConfigSourceSet(this.getClass().getSimpleName() + ": " + new Random().nextLong()); + private HandlersConfigurerDi configurer; + + // TODO: Remove once tests use ConfigSet rather than dir: + private final static String testFiles[] = { + "components.cfg", + "handlers.cfg", + "bundles.cfg", + "string.cfg", + "int.cfg", + "renderers.cfg", + "diagnostics.cfg", + "qr-templates.cfg", + "documentmanager.cfg", + "schemamapping.cfg", + "chains.cfg", + "container-mbus.cfg", + "container-mbus.cfg", + "specialtokens.cfg", + "documentdb-info.cfg", + "qr-search.cfg", + "query-profiles.cfg" + }; + private final Set<File> createdFiles = new LinkedHashSet<>(); + private int lastGeneration = 0; + private final Container container; + + private void createFiles(String configId) { + if (configId.startsWith("dir:")) { + try { + System.setProperty("config.id", configId); + String dirName = configId.substring(4); + for (String file : testFiles) { + createIfNotExists(dirName, file); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + // TODO: Remove once tests use ConfigSet rather than dir: + private void createIfNotExists(String dir, String file) throws IOException { + final File f = new File(dir + "/" + file); + if (f.createNewFile()) { + createdFiles.add(f); + } + } + + public HandlersConfigurerTestWrapper(String configId) { + this(Container.get(), configId); + } + + public HandlersConfigurerTestWrapper(Container container, String configId) { + createFiles(configId); + MockOsgi mockOsgi = new MockOsgi(); + container.setOsgi(mockOsgi); + ComponentDeconstructor testDeconstructor = getTestDeconstructor(); + configurer = new HandlersConfigurerDi( + new CloudSubscriberFactory(configSources), + container, + configId, + testDeconstructor, + Guice.createInjector(), + mockOsgi); + this.container = container; + } + + private ComponentDeconstructor getTestDeconstructor() { + return new ComponentDeconstructor() { + @Override + public void deconstruct(Object component) { + if (component instanceof AbstractComponent) { + AbstractComponent abstractComponent = (AbstractComponent) component; + if (abstractComponent.isDeconstructable()) + ((AbstractComponent) component).deconstruct(); + } + }}; + } + + public void reloadConfig() { + configurer.reloadConfig(++lastGeneration); + try { + configurer.runOnceAndEnsureRegistryHackRun(Guice.createInjector()); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + public void shutdown() { + // TODO: Remove once tests use ConfigSet rather than dir: + for (File f : createdFiles) { + f.delete(); + } + } + + public ComponentRegistry<RequestHandler> getRequestHandlerRegistry() { + return container.getRequestHandlerRegistry(); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/document/package-info.java b/container-core/src/main/java/com/yahoo/container/core/document/package-info.java new file mode 100644 index 00000000000..ffb130493ed --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/document/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.core.document; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/core/http/package-info.java b/container-core/src/main/java/com/yahoo/container/core/http/package-info.java new file mode 100644 index 00000000000..2db74625480 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/http/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.core.http; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/core/package-info.java b/container-core/src/main/java/com/yahoo/container/core/package-info.java new file mode 100644 index 00000000000..c9b43a8be66 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.core; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/core/slobrok/SlobrokConfigurator.java b/container-core/src/main/java/com/yahoo/container/core/slobrok/SlobrokConfigurator.java new file mode 100644 index 00000000000..7e19df254c1 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/slobrok/SlobrokConfigurator.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.core.slobrok; + +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.cloud.config.SlobroksConfig.Slobrok; +import com.yahoo.container.Container; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Configures which slobrok nodes the container should register with. + * @author tonytv + */ +public class SlobrokConfigurator { + public SlobrokConfigurator(SlobroksConfig config) { + Container.get().getRpcAdaptor().registerInSlobrok( + connectionSpecs(config.slobrok())); + } + + private static List<String> connectionSpecs(List<Slobrok> slobroks) { + return slobroks.stream(). + map(Slobrok::connectionspec). + collect(Collectors.toList()); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java new file mode 100644 index 00000000000..368a1f2cbf6 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.container.logging.CircularArrayAccessLogKeeper; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.concurrent.Executor; + +/** + * Exposes access log through http. + * + * @author dybdahl + */ +public class AccessLogRequestHandler extends ThreadedHttpRequestHandler { + private final CircularArrayAccessLogKeeper circularArrayAccessLogKeeper; + private final JsonFactory jsonFactory = new JsonFactory(); + + public AccessLogRequestHandler(Executor executor, CircularArrayAccessLogKeeper circularArrayAccessLogKeeper) { + super(executor); + this.circularArrayAccessLogKeeper = circularArrayAccessLogKeeper; + } + + @Override + public HttpResponse handle(HttpRequest request) { + final List<String> uris = circularArrayAccessLogKeeper.getUris(); + + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + + JsonGenerator generator = jsonFactory.createJsonGenerator(outputStream); + generator.writeStartObject(); + generator.writeArrayFieldStart("entries"); + for (String uri : uris) { + generator.writeStartObject(); + generator.writeStringField("url", uri); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + generator.close(); + } + }; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java new file mode 100644 index 00000000000..351af7f2563 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler; + + +/** + * The coverage report for a result set. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author balder + */ +public class Coverage { + + protected long docs; + protected long active; + protected int nodes; + protected int resultSets; + protected int fullResultSets; + + // need a default setting for deserialization logic in subclasses + protected FullCoverageDefinition fullReason = FullCoverageDefinition.DOCUMENT_COUNT; + + protected enum FullCoverageDefinition { + EXPLICITLY_FULL, EXPLICITLY_INCOMPLETE, DOCUMENT_COUNT; + } + + /** + * Build an invalid instance to initiate manually. + */ + protected Coverage() { + } + + protected Coverage(long docs, long active, int nodes, int resultSets) { + this(docs, active, nodes, resultSets, FullCoverageDefinition.DOCUMENT_COUNT); + } + + public Coverage(long docs, int nodes, boolean full) { + this(docs, nodes, full, 1); + } + + public Coverage(long docs, int nodes, boolean full, int resultSets) { + this(docs, docs, nodes, resultSets, full ? FullCoverageDefinition.EXPLICITLY_FULL + : FullCoverageDefinition.EXPLICITLY_INCOMPLETE); + } + + private Coverage(long docs, long active, int nodes, int resultSets, FullCoverageDefinition fullReason) { + this.docs = docs; + this.nodes = nodes; + this.active = active; + this.resultSets = resultSets; + this.fullReason = fullReason; + this.fullResultSets = getFull() ? resultSets : 0; + } + + public void merge(Coverage other) { + if (other == null) { + return; + } + docs += other.getDocs(); + nodes += other.getNodes(); + active += other.getActive(); + resultSets += other.getResultSets(); + fullResultSets += other.getFullResultSets(); + + // explicitly incomplete beats doc count beats explicitly full + switch (other.fullReason) { + case EXPLICITLY_FULL: + // do nothing + break; + case EXPLICITLY_INCOMPLETE: + fullReason = FullCoverageDefinition.EXPLICITLY_INCOMPLETE; + break; + case DOCUMENT_COUNT: + if (fullReason == FullCoverageDefinition.EXPLICITLY_FULL) { + fullReason = FullCoverageDefinition.DOCUMENT_COUNT; + } + break; + } + } + + /** + * The number of documents searched for this result. If the final result + * set is produced through several queries, this number will be the sum + * for all the queries. + */ + public long getDocs() { + return docs; + } + + /** + * Total number of documents that could be searched. + * + * @return Total number of active documents + */ + public long getActive() { return active; } + + /** + * @return whether the search had full coverage or not + */ + public boolean getFull() { + switch (fullReason) { + case EXPLICITLY_FULL: + return true; + case EXPLICITLY_INCOMPLETE: + return false; + case DOCUMENT_COUNT: + return docs == active; + default: + throw new IllegalStateException("Implementation out of sync. Please report this as a bug."); + } + } + + /** + * @return the number of search instances which participated in the search. + */ + public int getNodes() { + return nodes; + } + + /** + * A Coverage instance contains coverage information for potentially more + * than one search. If several queries, e.g. through blending of results + * from multiple clusters, produced a result set, this number will show how + * many of the result sets for these queries had full coverage. + * + * @return the number of result sets which had full coverage + */ + public int getFullResultSets() { + return fullResultSets; + } + + /** + * A Coverage instance contains coverage information for potentially more + * than one search. If several queries, e.g. through blending of results + * from multiple clusters, produced a result set, this number will show how + * many result sets containing coverage information this Coverage instance + * contains information about. + * + * @return the number of result sets with coverage information for this instance + */ + public int getResultSets() { + return resultSets; + } + + /** + * An int between 0 (inclusive) and 100 (inclusive) representing how many + * percent coverage the result sets this Coverage instance contains information + * about had. + */ + public int getResultPercentage() { + if (getResultSets() == 0) { + return 0; + } + if (docs < active) { + return (int) (docs * 100 / active); + } + return getFullResultSets() * 100 / getResultSets(); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/Prefix.java b/container-core/src/main/java/com/yahoo/container/handler/Prefix.java new file mode 100644 index 00000000000..84d4043f11c --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/Prefix.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler; + +/** + * Wrapper to maintain indirections between prefixes and Handler instances. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class Prefix implements Comparable<Prefix> { + + public final String prefix; + public final String[] elements; + public final String handler; + + public Prefix(String prefix, String handler) { + super(); + this.prefix = prefix; + this.elements = prefix.split("/"); + this.handler = handler; + } + + public Prefix(String prefix) { + this(prefix, null); + } + + public boolean hasAnyCommonPrefix(String router) { + return router.codePointAt(0) == prefix.codePointAt(0); + } + + @Override + public int compareTo(Prefix other) { + return prefix.compareTo(other.prefix); + } + + public boolean matches(String path) { + String[] pathElements = path.split("/"); + if (pathElements.length < elements.length) { + return false; + } + for (int i = 0; i < elements.length; ++i) { + if (!(elements[i].equals(pathElements[i]))) { + return false; + } + } + return true; + } + + @Override + public String toString() { + return prefix + ": " + handler; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java new file mode 100644 index 00000000000..51e1c0c1d53 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -0,0 +1,157 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import com.google.inject.Inject; +import com.yahoo.container.protect.ProcessTerminator; +import com.google.common.util.concurrent.ForwardingExecutorService; +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.jdisc.Metric; + +/** + * A configurable thread pool provider. This provides the worker threads used for normal request processing. + * Request an Executor injected in your component constructor if you want to use it. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @author bratseth + */ +public class ThreadPoolProvider extends AbstractComponent implements Provider<Executor> { + + private final ExecutorServiceWrapper threadpool; + + @Inject + public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric) { + this(threadpoolConfig, metric, new ProcessTerminator()); + } + + public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric, ProcessTerminator processTerminator) { + WorkerCompletionTimingThreadPoolExecutor executor = + new WorkerCompletionTimingThreadPoolExecutor(threadpoolConfig.maxthreads(), + threadpoolConfig.maxthreads(), + 0L, TimeUnit.SECONDS, + new SynchronousQueue<>(false), + ThreadFactoryFactory.getThreadFactory("threadpool")); + // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also + // get the dreaded thread locals initialized even if they will never run. + // That counters what we we want to achieve with the Q that will prefer thread locality. + executor.prestartAllCoreThreads(); + threadpool = new ExecutorServiceWrapper(executor, metric, processTerminator, + threadpoolConfig.maxThreadExecutionTimeSeconds() * 1000L); + } + + /** + * Get the Executor provided by this class. This Executor will by default + * also be used for search queries and processing requests. + * + * @return a possibly shared executor + */ + @Override + public Executor get() { return threadpool; } + + /** + * Shutdown the thread pool, give a grace period of 1 second before forcibly + * shutting down all worker threads. + */ + @Override + public void deconstruct() { + boolean terminated; + + super.deconstruct(); + threadpool.shutdown(); + try { + terminated = threadpool.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + if (!terminated) { + threadpool.shutdownNow(); + } + } + + /** + * A service executor wrapper which emits metrics and + * shuts down the vm when no workers are available for too long to avoid containers lingering in a blocked state. + */ + private final static class ExecutorServiceWrapper extends ForwardingExecutorService { + + private final WorkerCompletionTimingThreadPoolExecutor wrapped; + private final Metric metric; + private final ProcessTerminator processTerminator; + private final long maxThreadExecutionTimeMillis; + + private ExecutorServiceWrapper(WorkerCompletionTimingThreadPoolExecutor wrapped, + Metric metric, ProcessTerminator processTerminator, + long maxThreadExecutionTimeMillis) { + this.wrapped = wrapped; + this.metric = metric; + this.processTerminator = processTerminator; + this.maxThreadExecutionTimeMillis = maxThreadExecutionTimeMillis; + } + + /** + * Tracks all instances of RejectedExecutionException. + * ThreadPoolProvider returns an executor, so external uses will not + * have access to the methods declared by ExecutorService. + * (execute(Runnable) is declared by Executor.) + */ + @Override + public void execute(Runnable command) { + try { + metric.set(MetricNames.THREAD_POOL_SIZE, wrapped.getPoolSize(), null); + metric.set(MetricNames.ACTIVE_THREADS, wrapped.getActiveCount(), null); + super.execute(command); + } catch (RejectedExecutionException e) { + metric.add(MetricNames.REJECTED_REQUEST, 1, null); + long timeSinceLastReturnedThreadMillis = System.currentTimeMillis() - wrapped.lastThreadReturnTimeMillis; + if (timeSinceLastReturnedThreadMillis > maxThreadExecutionTimeMillis) + processTerminator.logAndDie("No worker threads have been available for " + + timeSinceLastReturnedThreadMillis + " ms. Shutting down.", true); + throw e; + } + } + + @Override + protected ExecutorService delegate() { return wrapped; } + + private static final class MetricNames { + private static final String REJECTED_REQUEST = "serverRejectedRequests"; + private static final String THREAD_POOL_SIZE = "serverThreadPoolSize"; + private static final String ACTIVE_THREADS = "serverActiveThreads"; + } + + } + + /** A thread pool executor which maintains the last time a worker completed */ + private final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { + + volatile long lastThreadReturnTimeMillis = System.currentTimeMillis(); + + public WorkerCompletionTimingThreadPoolExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue<Runnable> workQueue, + ThreadFactory threadFactory) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); + } + + @Override + protected void afterExecute(Runnable r, Throwable t) { + lastThreadReturnTimeMillis = System.currentTimeMillis(); + } + + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/Timing.java b/container-core/src/main/java/com/yahoo/container/handler/Timing.java new file mode 100644 index 00000000000..7c9fd862ef3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/Timing.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler; + + +/** + * <p>A wrapper for timing of events in the course of a query evaluation. Advanced + * database searches and similar could use these structures as well.</p> + * + * <p>Not adding this object will lead to less exact entries in the query + * log. It is legal to set only queryStartTime and set the other values + * to zero.</p> + * + * <p>If you do not understand the fields, just avoid creating this object + * in you handler.</p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Timing { + + protected long summaryStartTime; + + protected long queryStartTime; + + protected long timeout; + + /** + * Do consider using + * com.yahoo.search.handler.SearchResponse.createTiming(Query, Result) if + * instead of this constructor if you are creating a Timing instance in a + * search context. + * + * @param summaryStartTime when fetching of document contents started + * @param queryStartTime when the request started + * @param timeout maximum allowed lifetime of the request + */ + public Timing(long summaryStartTime, long ignored, long queryStartTime, long timeout) { + super(); + this.summaryStartTime = summaryStartTime; + this.queryStartTime = queryStartTime; + this.timeout = timeout; + } + + /** + * Summary start time is when the fetching of hit/document contents + * start. (As opposed to just analyzing hit relevancies.) + * + * @return the start time of summary fetching or 0 + */ + public long getSummaryStartTime() { + return summaryStartTime; + } + + /** + * This is the start of the server's evaluation of a query + * or request, after full reception of it through the network. + * It will usually be intialized implicitly from the value + * generated by the com.yahoo.search.Query constructor. + * + * @return the starting time of query construction + */ + public long getQueryStartTime() { + return queryStartTime; + } + + /** + * This is the timeout that was given to this query. + * + * @return The timeout given allowed to the query. + */ + public long getTimeout() { + return timeout; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java new file mode 100644 index 00000000000..524cbc52dd5 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler; + +import java.util.IdentityHashMap; +import java.util.Map; + +import com.google.inject.Inject; +import com.yahoo.container.QrSearchersConfig; + +/** + * API for programmatically removing the container from VIP rotation. + * + * @author steinar + */ +public class VipStatus { + private final Map<Object, Boolean> clusters = new IdentityHashMap<>(); + + public VipStatus() { + this(null); + } + + @Inject + public VipStatus(QrSearchersConfig dispatchers) { + // the config is not used for anything, it's just a dummy to create a + // dependency link to which dispatchers are used + } + + /** + * Set a service or cluster into rotation. + * + * @param clusterIdentifier + * an object where the object identity will serve to identify the + * cluster or service + */ + public void addToRotation(Object clusterIdentifier) { + synchronized (clusters) { + clusters.put(clusterIdentifier, Boolean.TRUE); + } + } + + /** + * Set a service or cluster out of rotation. + * + * @param clusterIdentifier + * an object where the object identity will serve to identify the + * cluster or service + */ + public void removeFromRotation(Object clusterIdentifier) { + synchronized (clusters) { + clusters.put(clusterIdentifier, Boolean.FALSE); + } + } + + /** + * Tell whether the container is connected to any active services at all. + * + * @return true if at least one service or cluster is up, or if no services + * are registered (yet) + */ + public boolean isInRotation() { + synchronized (clusters) { + // if no stored state, try serving + if (clusters.size() == 0) { + return true; + } + for (Boolean inRotation : clusters.values()) { + if (inRotation) { + return true; + } + } + } + return false; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java new file mode 100644 index 00000000000..3be1d08c5dc --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -0,0 +1,195 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.concurrent.Executor; +import java.util.logging.Logger; + +import com.google.inject.Inject; +import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Metric; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.defaults.Defaults; + +/** + * Transmit status to VIP from file or memory. Bind this to + * "http://{@literal *}/status.html" to serve VIP status requests. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class VipStatusHandler extends ThreadedHttpRequestHandler { + + private static final Logger log = Logger.getLogger(VipStatusHandler.class.getName()); + + private static final String NUM_REQUESTS_METRIC = "jdisc.http.requests.status"; + + private final boolean accessDisk; + private final File statusFile; + private final VipStatus vipStatus; + private final boolean noSearchBackendsImpliesOutOfService; + + private volatile boolean previouslyInRotation = true; + + // belongs in the response, but that's not a static class + static final String OK_MESSAGE = "<title>OK</title>\n"; + static final byte[] VIP_OK = Utf8.toBytes(OK_MESSAGE); + + class StatusResponse extends HttpResponse { + + static final String COULD_NOT_FIND_STATUS_FILE = "Could not find status file.\n"; + static final String NO_SEARCH_BACKENDS = "No search backends available, VIP status disabled."; + private static final String TEXT_HTML = "text/html"; + private String contentType = TEXT_HTML; + private byte[] data = null; + private File file = null; + + private StatusResponse() { + super(com.yahoo.jdisc.http.HttpResponse.Status.OK); // status may be overwritten below + if (noSearchBackendsImpliesOutOfService && !vipStatus.isInRotation()) { + searchContainerOutOfService(); + } else if (accessDisk) { + preSlurpFile(); + } else { + vipRespond(); + } + } + + @Override + public void render(OutputStream stream) throws IOException { + if (file != null) { + readAndWrite(stream); + } + else if (data != null) { + stream.write(data); + } + else { + throw new IllegalStateException( + "Neither file nor hardcoded data. This is a bug, please notify the Vespa team."); + } + stream.close(); + } + + private void readAndWrite(OutputStream stream) throws IOException { + InputStream input; + int lastRead = 0; + input = new FileInputStream(file); + try { + while (lastRead != -1) { + byte[] buffer = new byte[5000]; + lastRead = input.read(buffer); + if (lastRead > 0) { + stream.write(buffer, 0, lastRead); + } + } + } finally { + stream.close(); + input.close(); + } + } + + private void preSlurpFile() { + try { + if (!statusFile.exists()) { + fileNotFound(); + return; + } + if (!statusFile.canRead()) { + accessDenied(); + return; + } + } catch (SecurityException e) { + internalError(); + return; + } + this.file = statusFile; + } + + private void accessDenied() { + contentType = "text/plain"; + data = Utf8.toBytes("Status file inaccessible.\n"); + setStatus(com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND); + } + + private void internalError() { + contentType = "text/plain"; + data = Utf8.toBytes("Internal error while fetching status file.\n"); + setStatus(com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND); + } + + private void fileNotFound() { + contentType = "text/plain"; + data = Utf8.toBytes(COULD_NOT_FIND_STATUS_FILE); + setStatus(com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND); + } + + private void vipRespond() { + data = VIP_OK; + } + + /** + * Behaves like a VIP status response file has been deleted. + */ + private void searchContainerOutOfService() { + contentType = "text/plain"; + data = Utf8.toBytes(NO_SEARCH_BACKENDS); + setStatus(com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND); + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public String getCharacterEncoding() { + return null; + } + } + + public VipStatusHandler(Executor executor, VipStatusConfig vipConfig, Metric metric) { + this(executor, vipConfig, metric, null); + } + + @Inject + public VipStatusHandler(Executor executor, VipStatusConfig vipConfig, Metric metric, VipStatus vipStatus) { + super(executor, metric); + this.accessDisk = vipConfig.accessdisk(); + this.statusFile = new File(Defaults.getDefaults().underVespaHome(vipConfig.statusfile())); + this.noSearchBackendsImpliesOutOfService = vipConfig.noSearchBackendsImpliesOutOfService(); + this.vipStatus = vipStatus; + } + + @Override + public HttpResponse handle(HttpRequest request) { + if (metric != null) + metric.add(NUM_REQUESTS_METRIC, 1, null); + if (noSearchBackendsImpliesOutOfService) { + updateAndLogRotationState(); + } + return new StatusResponse(); + } + + private void updateAndLogRotationState() { + final boolean currentlyInRotation = vipStatus.isInRotation(); + final boolean previousRotationAnswer = previouslyInRotation; + previouslyInRotation = currentlyInRotation; + + if (previousRotationAnswer != currentlyInRotation) { + if (currentlyInRotation) { + log.log(LogLevel.INFO, "Putting container back into rotation by serving status.html again."); + } else { + log.log(LogLevel.WARNING, "Removing container from rotation by no longer serving status.html."); + } + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/observability/.gitignore b/container-core/src/main/java/com/yahoo/container/handler/observability/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/observability/.gitignore diff --git a/container-core/src/main/java/com/yahoo/container/handler/package-info.java b/container-core/src/main/java/com/yahoo/container/handler/package-info.java new file mode 100644 index 00000000000..6bc3276ba21 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Contains the handler framework of the container. + */ +@ExportPackage +@PublicApi +package com.yahoo.container.handler; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java new file mode 100644 index 00000000000..57e09aea81c --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.test; + +import com.google.common.annotations.Beta; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.jdisc.Metric; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * This is a generic http handler that can be used to mock a service when testing your application on jDISC. + * Configuration and necessary files are given to the handle in its configuration. + * + * Example config: + * <pre> + * <handler id="MockService"> + * <config name="container.handler.test.mockservice"> + * <file>myresponses.txt</file> + * </config> + * <binding>http://*\/my/service/path1/*</binding> + * </handler> + * </pre> + * + * The file formats supported out of the box is text, see {@link com.yahoo.container.handler.test.MockService.TextFileHandler}. + * for descriptions of the format. + * + * @author lulf + * @since 5.1.21 + */ +@Beta +public class MockService extends LoggingRequestHandler { + + private final static Logger log = Logger.getLogger(MockService.class.getName()); + private MockServiceHandler handler; + + /** + * Create a mock service that mocks an external service using data provided via file distribution. + * A custom handler can be created by subclassing and overriding the createHandler method. + * + * @param executor An {@link Executor} used to create threads. + * @param accessLog An {@link AccessLog} where requests will be logged. + * @param fileAcquirer A {@link FileAcquirer} which is used to fetch file from config. + * @param config A {@link MockserviceConfig} for this service. + * @throws InterruptedException if unable to get data file within timeout. + * @throws IOException if unable to create handler due to some IO errors. + */ + public MockService(Executor executor, AccessLog accessLog, FileAcquirer fileAcquirer, MockserviceConfig config, Metric metric) throws InterruptedException, IOException { + super(executor, accessLog, metric); + File dataFile = fileAcquirer.waitFor(config.file(), config.fileAcquirerTimeout(), TimeUnit.SECONDS); + this.handler = createHandler(dataFile); + } + + /** + * Create a handler for a file. Override this method to handle a custom file syntax of your own. + * + * @param dataFile A file to read. + * @return a {@link MockServiceHandler} used to handle requests. + * @throws IOException if errors occured when loading the file + */ + protected MockServiceHandler createHandler(File dataFile) throws IOException { + if (!dataFile.getName().endsWith(".txt")) { + throw new IllegalArgumentException("Default handler only support .txt files"); + } + return new TextFileHandler(dataFile); + } + + @Override + public final HttpResponse handle(HttpRequest request) { + try { + MockServiceHandler.Key key = handler.createKey(request); + MockServiceHandler.Value value = handler.get(key); + if (value == null) { + return new ErrorResponse(404, key + " was not found"); + } + return new RawResponse(value.returnCode, value.data, value.contentType); + } catch (Exception e) { + return new ExceptionResponse(500, e); + } + } + + /** + * A .txt file handler deals with the following format when reading data: + * method:url:responsecode:data + * + * For instance: + * GET:/my/path1:200:{\"foo\":\"bar\"} + * PUT:/my/path1:403:{\"error\":\"permission denied\"} + * TODO: Support binary files + */ + private static class TextFileHandler implements MockServiceHandler { + private final Map<MockServiceHandler.Key, Value> store = new HashMap<>(); + + public TextFileHandler(File dataFile) throws IOException { + BufferedReader reader = new BufferedReader(new FileReader(dataFile)); + readInputFile(reader); + } + + private void readInputFile(BufferedReader reader) throws IOException { + StringBuilder sb = new StringBuilder(); + int ch; + char prevChar = 0; + while ((ch = reader.read()) >= 0) { + char c = (char) ch; + if (prevChar == '\n') { + if (c == '\n') { + parseEntry(sb.toString()); + sb = new StringBuilder(); + prevChar = 0; + continue; + } else { + sb.append(prevChar); + } + } + if (c != '\n') { + sb.append(c); + } + prevChar = c; + } + parseEntry(sb.toString()); + } + + private void parseEntry(String entry) { + String [] components = entry.split(":", 4); + MockServiceHandler.Key key = new TextKey(com.yahoo.jdisc.http.HttpRequest.Method.valueOf(components[0]), components[1]); + Value value = new Value(Integer.parseInt(components[2]), components[3].getBytes(), "text/plain"); + store.put(key, value); + } + + @Override + public MockServiceHandler.Key createKey(HttpRequest request) { + StringBuilder sb = new StringBuilder(); + sb.append(request.getUri().getPath()); + String query = request.getUri().getQuery(); + if (query != null) { + sb.append("?").append(query); + } + return new TextKey(request.getMethod(), sb.toString()); + } + + @Override + public Value get(MockServiceHandler.Key key) { + return store.get(key); + } + } + + private static class RawResponse extends HttpResponse { + private final String contentType; + private final byte[] data; + RawResponse(int status, byte[] data, String contentType) { + super(status); + this.data = data; + this.contentType = contentType; + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(data); + } + } + + private static class ErrorResponse extends RawResponse { + ErrorResponse(int status, String message) { + super(status, message.getBytes(), "text/plain"); + } + } + + private static final class TextKey implements MockServiceHandler.Key { + private final com.yahoo.jdisc.http.HttpRequest.Method method; + private final String path; + public TextKey(com.yahoo.jdisc.http.HttpRequest.Method method, String path) { + this.method = method; + this.path = path; + } + + @Override + public int hashCode() { + return path.hashCode() + method.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other.getClass() != TextKey.class) { + return false; + } + TextKey rhs = (TextKey) other; + return (this.method == rhs.method) && + path.equals(rhs.path); + } + + @Override + public String toString() { + return method.toString() + ":" + path; + } + } + + private class ExceptionResponse extends HttpResponse { + private final Exception e; + public ExceptionResponse(int code, Exception e) { + super(code); + this.e =e; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + try (PrintStream ps = new PrintStream(outputStream)) { + e.printStackTrace(ps); + } + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/test/MockServiceHandler.java b/container-core/src/main/java/com/yahoo/container/handler/test/MockServiceHandler.java new file mode 100644 index 00000000000..45de04db2ea --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/test/MockServiceHandler.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.test; + +import com.google.common.annotations.Beta; +import com.yahoo.container.jdisc.HttpRequest; + +/** + * A service handler that is able to map a request to a key and retrieve a value given a key. + * + * @author lulf + * @since 5.1.21 + */ +@Beta +public interface MockServiceHandler { + /** + * Create a custom Key given a http request. This will be called for each request, and allows a handler + * to customize its key format. + * @param request The client http request. + * @return a {@link Key} used to query for the value. + */ + public Key createKey(HttpRequest request); + + /** + * Lookup a {@link Value} for a {@link Key}. Returns null if the key is not found. + * + * @param key The {@link Key} to look up. + * @return A {@link Value} used as response. + */ + public Value get(Key key); + + public final class Value { + public final int returnCode; + public final byte[] data; + public final String contentType; + + public Value(int returnCode, byte[] data, String contentType) { + this.returnCode = returnCode; + this.data = data; + this.contentType = contentType; + } + } + + public interface Key { + public int hashCode(); + public boolean equals(Object other); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/test/package-info.java b/container-core/src/main/java/com/yahoo/container/handler/test/package-info.java new file mode 100644 index 00000000000..7ecdaa57ef0 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/test/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Contains the handler test utilities. + */ +@ExportPackage +package com.yahoo.container.handler.test; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/http/AccessLogUtil.java b/container-core/src/main/java/com/yahoo/container/http/AccessLogUtil.java new file mode 100644 index 00000000000..4df202c05ff --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/http/AccessLogUtil.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.http; + +import com.yahoo.jdisc.http.HttpRequest; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.List; + +/** + * @author <a href="mailto:bakksjo@yahoo-inc.com">Oyvind Bakksjo</a> + */ +public class AccessLogUtil { + public static String getHttpMethod(final HttpRequest httpRequest) { + return httpRequest.getMethod().toString(); + } + + public static URI getUri(final HttpRequest httpRequest) { + return httpRequest.getUri(); + } + + public static String getHttpVersion(final HttpRequest httpRequest) { + return httpRequest.getVersion().toString(); + } + + public static String getReferrerHeader(final HttpRequest httpRequest) { + // Yes, the header name is misspelled in the standard + return getFirstHeaderValue(httpRequest, "Referer"); + } + + public static String getUserAgentHeader(final HttpRequest httpRequest) { + return getFirstHeaderValue(httpRequest, "User-Agent"); + } + + public static InetSocketAddress getRemoteAddress(final HttpRequest httpRequest) { + return (InetSocketAddress) httpRequest.getRemoteAddress(); + } + + private static String getFirstHeaderValue(final HttpRequest httpRequest, final String headerName) { + final List<String> headerValues = httpRequest.headers().get(headerName); + return (headerValues == null || headerValues.isEmpty()) ? "" : headerValues.get(0); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/http/BenchmarkingHeaders.java b/container-core/src/main/java/com/yahoo/container/http/BenchmarkingHeaders.java new file mode 100644 index 00000000000..bb3afc001ec --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/http/BenchmarkingHeaders.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.http; + +/** + * Class containing the names of all benchmarking getHeaders in request and response + * + * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + */ +public class BenchmarkingHeaders { + + public static final String REQUEST = "X-Yahoo-Vespa-Benchmarkdata"; + public static final String REQUEST_COVERAGE = "X-Yahoo-Vespa-Benchmarkdata-Coverage"; + + public static final String NUM_HITS = "X-Yahoo-Vespa-NumHits"; + public static final String NUM_FASTHITS = "X-Yahoo-Vespa-NumFastHits"; + public static final String NUM_GROUPHITS = "X-Yahoo-Vespa-NumGroupHits"; + public static final String NUM_ERRORS = "X-Yahoo-Vespa-NumErrors"; + public static final String TOTAL_HIT_COUNT = "X-Yahoo-Vespa-TotalHitCount"; + public static final String NUM_DOCSUMS = "X-Yahoo-Vespa-NumDocsums"; + public static final String QUERY_HITS = "X-Yahoo-Vespa-QueryHits"; + public static final String QUERY_OFFSET = "X-Yahoo-Vespa-QueryOffset"; + public static final String SEARCH_TIME = "X-Yahoo-Vespa-SearchTime"; + public static final String ATTR_TIME = "X-Yahoo-Vespa-AttributeFetchTime"; + public static final String FILL_TIME = "X-Yahoo-Vespa-FillTime"; + public static final String DOCS_SEARCHED = "X-Yahoo-Vespa-DocsSearched"; + public static final String NODES_SEARCHED = "X-Yahoo-Vespa-NodesSearched"; + public static final String FULL_COVERAGE = "X-Yahoo-Vespa-FullCoverage"; + +} diff --git a/container-core/src/main/java/com/yahoo/container/http/package-info.java b/container-core/src/main/java/com/yahoo/container/http/package-info.java new file mode 100644 index 00000000000..b356d9ce963 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/http/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.container.http; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/AsyncHttpResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/AsyncHttpResponse.java new file mode 100644 index 00000000000..851cf60737b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/AsyncHttpResponse.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import java.io.IOException; +import java.io.OutputStream; + +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; + +/** + * HTTP response which supports async response rendering. + * + * @author bratseth + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public abstract class AsyncHttpResponse extends HttpResponse { + + /** + * Create a new HTTP response with support for async output. + * + * @param status the HTTP status code for jdisc + * @see Response + */ + public AsyncHttpResponse(int status) { + super(status); + } + + /** + * Render to output asynchronously. The output stream will not be closed + * when this return. The implementation is responsible for closing the + * output (using the provided channel and completion handler) when (async) + * rendering is completed. + * + * @param output + * the stream to which content should be rendered + * @param networkChannel + * the channel which must be closed on completion + * @param handler + * the completion handler to submit when closing the channel, may + * be null + */ + public abstract void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) + throws IOException; + + /** + * Throws UnsupportedOperationException. Use + * {@link #render(OutputStream, ContentChannel, CompletionHandler)} instead. + */ + @Override + public final void render(OutputStream output) { + throw new UnsupportedOperationException("Illegal use."); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java new file mode 100644 index 00000000000..fbae1783805 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.io.BufferChain; +import com.yahoo.io.WritableByteTransmitter; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A buffered stream wrapping a ContentChannel. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ContentChannelOutputStream extends OutputStream implements WritableByteTransmitter { + + private static final Logger log = Logger.getLogger(ContentChannelOutputStream.class.getName()); + private final BufferChain buffer; + private final ContentChannel endpoint; + private long byteBufferData = 0L; + private boolean failed = false; + private final Object failLock = new Object(); + + public ContentChannelOutputStream(final ContentChannel endpoint) { + this.endpoint = endpoint; + buffer = new BufferChain(this); + } + + /** + * Buffered write of a single byte. + */ + @Override + public void write(final int b) throws IOException { + try { + buffer.append((byte) b); + } catch (RuntimeException e) { + throw new IOException(Exceptions.toMessageString(e), e); + } + } + + /** + * Flush the internal buffers, does not touch the ContentChannel. + */ + @Override + public void close() throws IOException { + // the endpoint is closed in a finally{} block inside AbstractHttpRequestHandler + // this class should be possible to close willynilly as it is exposed to plug-ins + try { + buffer.flush(); + } catch (RuntimeException e) { + throw new IOException(Exceptions.toMessageString(e), e); + } + } + + /** + * Flush the internal buffers, does not touch the ContentChannel. + */ + @Override + public void flush() throws IOException { + try { + buffer.flush(); + } catch (RuntimeException e) { + throw new IOException(Exceptions.toMessageString(e), e); + } + } + + /** + * Buffered write of the contents of the array to this stream, + * <i>copying</i> the contents of the given array to this stream. + * It is in other words safe to recycle the array {@code b}. + */ + @Override + public void write(final byte[] b, final int off, final int len) + throws IOException { + nonCopyingWrite(Arrays.copyOfRange(b, off, off + len)); + } + + /** + * Buffered write the contents of the array to this stream, + * <i>copying</i> the contents of the given array to this stream. + * It is in other words safe to recycle the array {@code b}. + */ + @Override + public void write(final byte[] b) throws IOException { + nonCopyingWrite(Arrays.copyOf(b, b.length)); + } + + /** + * Buffered write of the contents of the array to this stream, + * <i>transferring</i> ownership of that array to this stream. It is in + * other words <i>not</i> safe to recycle the array {@code b}. + */ + public void nonCopyingWrite(final byte[] b, final int off, final int len) + throws IOException { + try { + buffer.append(b, off, len); + } catch (RuntimeException e) { + throw new IOException(Exceptions.toMessageString(e), e); + } + } + + /** + * Buffered write the contents of the array to this stream, + * <i>transferring</i> ownership of that array to this stream. It is in + * other words <i>not</i> safe to recycle the array {@code b}. + */ + public void nonCopyingWrite(final byte[] b) throws IOException { + try { + buffer.append(b); + } catch (RuntimeException e) { + throw new IOException(Exceptions.toMessageString(e), e); + } + } + + + /** + * Write a ByteBuffer to the wrapped ContentChannel. Do invoke + * {@link ContentChannelOutputStream#flush()} before send(ByteBuffer) to + * avoid garbled output if the stream API has been accessed before using the + * ByteBuffer based API. As with ContentChannel, this transfers ownership of + * the ByteBuffer to this stream. + */ + @Override + public void send(final ByteBuffer src) throws IOException { + // Don't do a buffer.flush() from here, this method is used by the + // buffer itself + try { + byteBufferData += (long) src.remaining(); + endpoint.write(src, new LoggingCompletionHandler()); + } catch (RuntimeException e) { + throw new IOException(Exceptions.toMessageString(e), e); + } + } + + /** + * Give the number of bytes written. + * + * @return the number of bytes written to this stream + */ + public long written() { + return buffer.appended() + byteBufferData; + } + + class LoggingCompletionHandler implements CompletionHandler { + @Override + public void completed() { + } + + @Override + public void failed(Throwable t) { + Level logLevel; + synchronized (failLock) { + if (failed) { + logLevel = LogLevel.SPAM; + } else { + logLevel = LogLevel.DEBUG; + } + failed = true; + } + if (log.isLoggable(logLevel)) { + log.log(logLevel, "Got exception when writing to client: " + Exceptions.toMessageString(t)); + } + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/EmptyResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/EmptyResponse.java new file mode 100644 index 00000000000..e4feb976af0 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/EmptyResponse.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Placeholder response when no content, only headers and status is to be + * returned. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class EmptyResponse extends HttpResponse { + + public EmptyResponse(int status) { + super(status); + } + + public void render(OutputStream outputStream) throws IOException { + // NOP + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ExtendedResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/ExtendedResponse.java new file mode 100644 index 00000000000..adfb461c122 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ExtendedResponse.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import java.io.IOException; +import java.io.OutputStream; + +import com.yahoo.container.handler.Coverage; +import com.yahoo.container.handler.Timing; +import com.yahoo.container.logging.HitCounts; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; + +/** + * An HTTP response supporting async rendering and extended information for + * logging. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public abstract class ExtendedResponse extends AsyncHttpResponse { + + public ExtendedResponse(int status) { + super(status); + } + + @Override + public abstract void render(OutputStream output, + ContentChannel networkChannel, CompletionHandler handler) + throws IOException; + + /** + * @return user name performing the request + */ + public String getUser() { + return null; + } + + /** + * The parsed query or some other normal form for the query/request + * resulting in this Response. Never null. This default implementation + * returns null though. + */ + public String getParsedQuery() { + return null; + } + + /** + * Returns timing information about the processing leading to this response. + * This default implementation returns null. + * + * @see com.yahoo.container.handler.Timing + * @return a Timing instance or null + */ + public Timing getTiming() { + return null; + } + + /** + * Returns the completeness of the scan of the total known data for this + * response. This default implementation returns null. + * + * @see Coverage + * @return coverage information or null + */ + public Coverage getCoverage() { + return null; + } + + /** + * Returns the number of "hits" in this. This default implementation returns + * null. + * + * @return a Counts instance or null + */ + public HitCounts getHitCounts() { + return null; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java new file mode 100644 index 00000000000..dadac8e8f16 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java @@ -0,0 +1,616 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.google.inject.Key; +import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.Container; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.ResourceReference; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.http.HttpRequest.Version; +import com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler; +import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.processing.request.Properties; + +import java.io.InputStream; +import java.net.SocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.jdisc.http.HttpRequest.Method; + +/** + * Wraps a JDisc HTTP request for a synchronous API. + * <p> + * The properties of this request represents what was received in the request + * and are thus immutable. If you need mutable abstractions, use a higher level + * framework, e.g. Processing. + * + * @author musum + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1 + */ +public class HttpRequest { + + private final com.yahoo.jdisc.http.HttpRequest parentRequest; + private final Map<String, String> properties; + private final InputStream requestData; + + /** + * Builder of HTTP requests + */ + public static class Builder { + + private final HttpRequest parent; + private com.yahoo.jdisc.http.HttpRequest jdiscRequest; + Method method = null; + Version version = null; + Map<String, String> properties = new HashMap<>(); + InputStream requestData = null; + URI uri = null; + CurrentContainer container = null; + private String nag = " must be set before the attempted operation."; + SocketAddress remoteAddress; + + private void boom(Object ref, String what) { + if (ref == null) { + throw new IllegalStateException(what + nag); + } + } + + private void requireUri() { + boom(uri, "An URI"); + } + + private void requireContainer() { + boom(container, "A CurrentContainer instance"); + } + + private void ensureJdiscParent() { + if (jdiscRequest == null) { + if (parent == null) { + throw new IllegalStateException("Neither another HttpRequest nor JDisc request available."); + } else { + jdiscRequest = parent.getJDiscRequest(); + } + } + } + + private void ensureRequestData() { + if (requestData == null) { + if (parent == null) { + throw new IllegalStateException( + "Neither another HttpRequest nor request data input stream available."); + } else { + requestData = parent.getData(); + } + } + } + + /** + * Instantiate a request builder with defaults from an existing request. + * If the request is null, a JDisc request must be set explitly using + * {@link #jdiscRequest(com.yahoo.jdisc.http.HttpRequest)} before + * instantiating any HTTP request. + * + * @param request + * source for defaults and parent JDisc request, may be null + * + * @see HttpRequest#createTestRequest(String, com.yahoo.jdisc.http.HttpRequest.Method) + */ + public Builder(HttpRequest request) { + this(request, request.getJDiscRequest()); + } + + /** + * Instantiate a request builder with defaults from an existing request. + * + * @param request + * parent JDisc request + * + * @see HttpRequest#createTestRequest(String, com.yahoo.jdisc.http.HttpRequest.Method) + */ + public Builder(com.yahoo.jdisc.http.HttpRequest request) { + this(null, request); + } + + private Builder(HttpRequest parent, com.yahoo.jdisc.http.HttpRequest jdiscRequest) { + this.parent = parent; + this.jdiscRequest = jdiscRequest; + populateProperties(); + + } + + private void populateProperties() { + if (parent == null) { + return; + } + + properties.putAll(parent.propertyMap()); + } + + /** + * Add a parameter to the request. Multi-value parameters are not + * supported. + * + * @param key + * parameter name + * @param value + * parameter value + * @return this Builder instance + */ + public Builder put(String key, String value) { + properties.put(key, value); + return this; + } + + /** + * Removes the parameter from the request properties. + * If there is no such parameter, nothing will be done. + */ + public Builder removeProperty(String parameterName) { + properties.remove(parameterName); + return this; + } + + /** + * Set the HTTP method for the new request. + * + * @param method + * the HTTP method to use for the new request + * @return this Builder instance + */ + public Builder method(Method method) { + this.method = method; + return this; + } + + /** + * Define the JDisc parent request. + * + * @param request + * a valid JDisc request for the current container + * @return this Builder instance + */ + public Builder jdiscRequest(com.yahoo.jdisc.http.HttpRequest request) { + this.jdiscRequest = request; + return this; + } + + /** + * Set an inputstream to use for the request. If not set, the data from + * the original HttpRequest is used. + * + * @param requestData data to be consumed, e.g. POST data + * @return this Builder instance + */ + public Builder requestData(InputStream requestData) { + this.requestData = requestData; + return this; + } + + /** + * Set the URI of the server request created. + * + * @param uri a valid URI for a server request + * @return this Builder instance + */ + public Builder uri(URI uri) { + this.uri = uri; + return this; + } + + /** + * Create a new HTTP request without creating a new JDisc request. This + * is for scenarios where another HTTP request handler is invoked + * directly without dispatching through JDisc. The parent JDisc request + * for the original HttpRequest will be passed on the new HttpRequest + * instance's JDisc request, but no properties will be propagated into + * the original JDisc request. + * + * @return a new HttpRequest instance reflecting the given request data + * and parameters + */ + public HttpRequest createDirectRequest() { + ensureRequestData(); + ensureJdiscParent(); + return new HttpRequest(jdiscRequest, requestData, properties); + } + + /** + * Start of API for synchronous HTTP request dispatch. Not yet ready for + * use. + * + * @return a new client request + */ + public HttpRequest createClientRequest() { + ensureJdiscParent(); + requireUri(); + com.yahoo.jdisc.http.HttpRequest clientRequest; + if (method == null) { + clientRequest = com.yahoo.jdisc.http.HttpRequest + .newClientRequest(jdiscRequest, uri); + } else { + if (version == null) { + clientRequest = com.yahoo.jdisc.http.HttpRequest + .newClientRequest(jdiscRequest, uri, method); + } else { + clientRequest = com.yahoo.jdisc.http.HttpRequest + .newClientRequest(jdiscRequest, uri, method, + version); + } + } + setParameters(clientRequest); + // TODO set requestData sanely + return new HttpRequest(clientRequest, requestData, properties); + } + + /** + * Start of API for synchronous HTTP request dispatch. Not yet ready for + * use. + * + * @return a new server request + */ + public HttpRequest createServerRequest() { + requireUri(); + requireContainer(); + com.yahoo.jdisc.http.HttpRequest serverRequest; + if (method == null) { + serverRequest = com.yahoo.jdisc.http.HttpRequest + .newServerRequest(container, uri); + } else { + if (version == null) { + serverRequest = com.yahoo.jdisc.http.HttpRequest + .newServerRequest(container, uri, method); + } else { + if (remoteAddress == null) { + serverRequest = com.yahoo.jdisc.http.HttpRequest + .newServerRequest(container, uri, method, + version); + } else { + serverRequest = com.yahoo.jdisc.http.HttpRequest + .newServerRequest(container, uri, method, + version, remoteAddress); + } + } + } + setParameters(serverRequest); + // TODO IO wiring + return new HttpRequest(serverRequest, requestData, properties); + } + + private void setParameters( + com.yahoo.jdisc.http.HttpRequest request) { + for (Map.Entry<String, String> entry : properties.entrySet()) { + request.parameters().put(entry.getKey(), wrap(entry.getValue())); + } + } + + } + + /** + * Wrap a JDisc HTTP request in a synchronous API. The properties from the + * JDisc request will be copied into the HTTP request. + * + * @param jdiscHttpRequest + * the JDisc request + * @param requestData + * the associated input stream, e.g. with POST request + */ + public HttpRequest(com.yahoo.jdisc.http.HttpRequest jdiscHttpRequest, InputStream requestData) { + this(jdiscHttpRequest, requestData, null); + } + + /** + * Wrap a JDisc HTTP request in a synchronous API. The properties from the + * JDisc request will be copied into the HTTP request. The mappings in + * propertyOverrides will mask the settings in the JDisc request. The + * content of propertyOverrides will be copied, so it is safe to re-use and + * changes in propertyOverrides after constructing the HttpRequest instance + * will obviously not be reflected by the request. The same applies for + * JDisc parameters. + * + * @param jdiscHttpRequest + * the JDisc request + * @param requestData + * the associated input stream, e.g. with POST request + * @param propertyOverrides + * properties which should not have the same settings as in the + * parent JDisc request, may be null + */ + public HttpRequest(com.yahoo.jdisc.http.HttpRequest jdiscHttpRequest, + InputStream requestData, Map<String, String> propertyOverrides) { + parentRequest = jdiscHttpRequest; + this.requestData = requestData; + properties = copyProperties(jdiscHttpRequest.parameters(), propertyOverrides); + } + + /** + * Create a new HTTP request from an URI. + * + * @param container the current container instance + * @param uri the request parameters + * @param method GET, POST, etc + * @param requestData the associated data stream, may be null + * @return a new HTTP request + */ + public static HttpRequest createRequest(CurrentContainer container, URI uri, + Method method, InputStream requestData) { + return createRequest(container, uri, method, requestData, null); + } + + /** + * Create a new HTTP request from an URI. + * + * @param container the current container instance + * @param uri the request parameters + * @param method GET, POST, etc + * @param requestData the associated data stream, may be null + * @param properties a set of properties to set in the request in addition to the implicit ones from the URI + * @return a new HTTP request + */ + public static HttpRequest createRequest(CurrentContainer container, + URI uri, Method method, InputStream requestData, + Map<String, String> properties) { + + final com.yahoo.jdisc.http.HttpRequest clientRequest = com.yahoo.jdisc.http.HttpRequest + .newClientRequest(new Request(container, uri), uri, method); + setProperties(clientRequest, properties); + return new HttpRequest(clientRequest, requestData); + } + + private static void setProperties(com.yahoo.jdisc.http.HttpRequest clientRequest, Map<String, String> properties) { + if (properties == null) { + return; + } + for (Map.Entry<String, String> entry : properties.entrySet()) { + clientRequest.parameters().put(entry.getKey(), wrap(entry.getValue())); + } + } + + // conservative code in case anything else depends on modifying these lists + private static List<String> wrap(String value) { + List<String> l = new ArrayList<>(4); + l.add(value); + return l; + } + + public static Optional<HttpRequest> getHttpRequest(com.yahoo.processing.Request processingRequest) { + final Properties requestProperties = processingRequest.properties(); + return Optional.ofNullable( + (HttpRequest) requestProperties.get(com.yahoo.processing.Request.JDISC_REQUEST)); + } + + public Optional<AccessLogEntry> getAccessLogEntry() { + return Optional.of(getJDiscRequest()) + .flatMap(AccessLoggingRequestHandler::getAccessLogEntry); + } + + private static URI createUri(String request) { + final URI uri; + try { + uri = new URI(request); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + return uri; + } + + /** + * Only for simpler unit testing. + * + * @param uri the complete URI string + * @param method POST, GET, etc + * @return a valid HTTP request + */ + public static HttpRequest createTestRequest(String uri, Method method) { + return createTestRequest(uri, method, null); + } + + /** + * Only for simpler unit testing. + * + * @param uri the complete URI string + * @param method POST, GET, etc + * @param requestData for simulating POST + * @return a valid HTTP request + */ + public static HttpRequest createTestRequest(String uri, Method method, InputStream requestData) { + return createTestRequest(uri, method, requestData, null); + } + + public static HttpRequest createTestRequest(String uri, Method method, InputStream requestData, Map<String, String> properties) { + return createRequest(new MockCurrentContainer(), createUri(uri), method, requestData, properties); + } + + private static Map<String, String> copyProperties(Map<String, List<String>> parameters, Map<String, String> parameterMask) { + Map<String, String> mask; + Map<String, String> view; + + if (parameterMask != null) { + mask = parameterMask; + } else { + mask = Collections.emptyMap(); + } + view = new HashMap<>(parameters.size() + mask.size()); + for (Map.Entry<String, List<String>> parameter : parameters.entrySet()) { + if (existsAsOriginalParameter(parameter.getValue())) { + List<String> values = parameter.getValue(); + view.put(parameter.getKey(), values.get(values.size() - 1)); // prefer the last value + } + } + view.putAll(mask); + return Collections.unmodifiableMap(view); + } + + private static boolean existsAsOriginalParameter(List<String> value) { + return value != null && value.size() > 0 && value.get(0) != null; + } + + /** + * Return the HTTP method (GET, POST...) of the incoming request. + * + * @return a Method instance matching the HTTP method of the request + */ + public Method getMethod() { + return parentRequest.getMethod(); + } + + /** + * Get the full URI corresponding to this request. + * + * @return the URI of this request + */ + public URI getUri() { + return parentRequest.getUri(); + } + + /** + * Access the underlying JDisc for this HTTP request. + * + * @return the corresponding JDisc request instance + */ + public com.yahoo.jdisc.http.HttpRequest getJDiscRequest() { + return parentRequest; + } + + /** + * Returns the value of a request property/parameter. + * Multi-value properties are not supported. + * + * @param name the name of the URI property to return + * @return the value of the property in question, or null if not present + */ + public String getProperty(String name) { + return properties.get(name); + } + + /** + * Return a read-only view of the request parameters. Multi-value parameters + * are not supported. + * + * @return a map containing all the parameters in the request + */ + public Map<String, String> propertyMap() { + return properties; + } + + /** + * Helper method to parse boolean request flags, using + * Boolean.parseBoolean(String). Unset values are regarded as false. + * + * @param name + * the name of a request property + * @return whether the property has been explicitly set to true + */ + public boolean getBooleanProperty(String name) { + if (getProperty(name) == null) { + return false; + } + return Boolean.parseBoolean(getProperty(name)); + } + + /** + * Check whether a property exists. + * + * @param name the name of a request property + * @return true if the property has a value + */ + public boolean hasProperty(String name) { + return properties.containsKey(name); + } + + /** + * Access an HTTP header in the request. Multi-value headers are not + * supported. + * + * @param name + * the name of an HTTP header + * @return the first pertinent value + */ + public String getHeader(String name) { + if (parentRequest.headers().get(name) == null) + return null; + return parentRequest.headers().get(name).get(0); + } + + /** + * Get the host segment of the URI of this request. + * + * @return the host name from the URI + */ + public String getHost() { + return getUri().getHost(); + } + + /** + * The port of the URI of this request. + * + * @return the port number of the URI + */ + public int getPort() { + return getUri().getPort(); + } + + /** + * The input stream for this request, i.e. data POSTed from the client. A + * client may read as much or as little data as needed from this stream, + * draining and closing will be done by the RequestHandler base classes + * using this HttpRequest (sub-)class. In other words, this stream should + * not be closed after use. + * + * @return the stream with the client data for this request + */ + public InputStream getData() { + return requestData; + } + + /** + * Helper class for testing only. + */ + private static class MockCurrentContainer implements CurrentContainer { + @Override + public Container newReference(URI uri) { + return new Container() { + + @Override + public RequestHandler resolveHandler(com.yahoo.jdisc.Request request) { + return null; + } + + @Override + public <T> T getInstance(Key<T> tKey) { + return null; + } + + @Override + public <T> T getInstance(Class<T> tClass) { + return null; + } + + @Override + public ResourceReference refer() { + return References.NOOP_REFERENCE; + } + + @Override + public void release() { + // NOP + } + + @Override + public long currentTimeMillis() { + return 0; + } + }; + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java new file mode 100644 index 00000000000..1bd8e3089a7 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpResponse.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.jdisc.Response; +import com.yahoo.processing.execution.Execution.Trace.LogValue; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; + +/** + * An HTTP response as an opaque payload with headers and content type. + * + * @author musum + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1 + */ +public abstract class HttpResponse { + + /** + * Default response content type; text/plain. + */ + public static final String DEFAULT_MIME_TYPE = "text/plain"; + + /** + * Default encoding/character set of a HTTP response; UTF-8. + */ + public static final String DEFAULT_CHARACTER_ENCODING = "UTF-8"; + + + private final Response parentResponse; + + /** + * Create a new HTTP response. + * + * @param status the HTTP status code to return with this response (may be changed later) + * @see Response + */ + public HttpResponse(int status) { + parentResponse = com.yahoo.jdisc.http.HttpResponse.newInstance(status); + } + + /** + * Marshal this response to the network layer. + */ + public abstract void render(OutputStream outputStream) throws IOException; + + /** + * The numeric HTTP status code, e.g. 200, 404 and so on. + * + * @return the numeric HTTP status code + */ + public int getStatus() { + return parentResponse.getStatus(); + } + + /** + * Sets the numeric HTTP status code this will return. + */ + public void setStatus(int status) { parentResponse.setStatus(status); } + + /** + * The response headers. + * + * @return a mutable, thread-unsafe view of the response headers + */ + public HeaderFields headers() { + return parentResponse.headers(); + } + + /** + * The underlying JDisc response. + * + * @return the actual response which will be used by the JDisc layer + */ + public com.yahoo.jdisc.Response getJdiscResponse() { + return parentResponse; + } + + /** + * The MIME type of the response contents or null. If null is returned, no + * content type header is added to the HTTP response. + * + * @return by default {@link HttpResponse#DEFAULT_MIME_TYPE} + */ + public String getContentType() { + return DEFAULT_MIME_TYPE; + } + + /** + * The name of the encoding for the response contents, if applicable. Return + * null if character set is not applicable to the response in question (e.g. + * binary formats). If null is returned, not "charset" element is added to + * the content type header. + * + * @return by default {@link HttpResponse#DEFAULT_CHARACTER_ENCODING} + */ + public String getCharacterEncoding() { + return DEFAULT_CHARACTER_ENCODING; + } + + // ======================================== + // Purely optional stuff from this point on + // ======================================== + + /** + * Override this method to add information from the response to the access log. + * + * Remember to also invoke super if you override it. + * + * @param accessLogEntry the access log entry to add information to. + */ + public void populateAccessLogEntry(final AccessLogEntry accessLogEntry) { + for (LogValue logValue : getLogValues()) { + accessLogEntry.addKeyValue(logValue.getKey(), logValue.getValue()); + } + } + + /** + * Complete creation of this response. + * This is called by the container once just before writing the response header back to the caller, + * so this is the last moment at which status and headers can be determined. + * This default implementation does nothing. + */ + public void complete() { } + + public Iterable<LogValue> getLogValues() { + return Collections::emptyIterator; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingCompletionHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingCompletionHandler.java new file mode 100644 index 00000000000..6ff20c8ccc3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingCompletionHandler.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.jdisc.handler.CompletionHandler; + +/** + * A completion handler which does access logging. + * + * @see ThreadedHttpRequestHandler#createLoggingCompletionHandler(long, long, HttpResponse, HttpRequest, ContentChannelOutputStream) + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public interface LoggingCompletionHandler extends CompletionHandler { + /** + * Set the commit start time to the current time. Commit start is only well + * defined for synchronous renderers, it is the point in time when rendering + * has finished, but there may still be I/O operations to transfer the data + * to the client pending. + */ + public void markCommitStart(); +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java new file mode 100644 index 00000000000..af478748eeb --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -0,0 +1,271 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.google.inject.Inject; +import com.yahoo.container.handler.Timing; +import com.yahoo.container.http.AccessLogUtil; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * A request handler base class extending the features of + * ThreadedHttpRequestHandler with access logging. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { + + private AccessLog accessLog; + + public LoggingRequestHandler(Executor executor, AccessLog accessLog) { + this(executor, accessLog, null); + } + + @Inject + public LoggingRequestHandler(Executor executor, AccessLog accessLog, Metric metric) { + this(executor, accessLog, metric, false); + } + + public LoggingRequestHandler(Executor executor, AccessLog accessLog, Metric metric, boolean allowAsyncResponse) { + super(executor, metric, allowAsyncResponse); + this.accessLog = accessLog; + } + + @Override + protected LoggingCompletionHandler createLoggingCompletionHandler( + long startTime, long renderStartTime, HttpResponse response, + HttpRequest httpRequest, ContentChannelOutputStream rendererWiring) { + return new LoggingHandler(startTime, renderStartTime, httpRequest, response, rendererWiring); + } + + private static String getClientIP(com.yahoo.jdisc.http.HttpRequest httpRequest) { + SocketAddress clientAddress = httpRequest.getRemoteAddress(); + if (clientAddress == null) + return "0.0.0.0"; + return clientAddress.toString(); + } + + private static long getEvalStart(Timing timing, long startTime) { + if (timing == null || timing.getQueryStartTime() == 0L) { + return startTime; + } else { + return timing.getQueryStartTime(); + } + } + + private static String remoteHostAddress( + com.yahoo.jdisc.http.HttpRequest httpRequest) { + SocketAddress remoteAddress = httpRequest.getRemoteAddress(); + if (remoteAddress == null) + return "0.0.0.0"; + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress() + .getHostAddress(); + } else { + throw new RuntimeException( + "Expected remote address of type InetSocketAddress, got " + + remoteAddress.getClass().getName()); + } + } + + private void logTimes(long startTime, String sourceIP, + long renderStartTime, long commitStartTime, long endTime, + String req, String normalizedQuery, Timing t) { + + // note: intentionally only taking time since request was received + long totalTime = endTime - startTime; + + long timeoutInterval = Long.MAX_VALUE; + long requestOverhead = 0; + long summaryStartTime = 0; + if (t != null) { + timeoutInterval = t.getTimeout(); + requestOverhead = t.getQueryStartTime() - startTime; + summaryStartTime = t.getSummaryStartTime(); + } + + if (totalTime <= timeoutInterval) { + return; + } + + StringBuilder msgbuf = new StringBuilder(); + msgbuf.append(normalizedQuery); + msgbuf.append(" from ").append(sourceIP).append(". "); + + if (requestOverhead > 0) { + msgbuf.append("Time from HTTP connection open to request reception "); + msgbuf.append(requestOverhead).append(" ms. "); + } + if (summaryStartTime != 0) { + msgbuf.append("Request time: "); + msgbuf.append(summaryStartTime - startTime).append(" ms. "); + msgbuf.append("Summary fetch time: "); + msgbuf.append(renderStartTime - summaryStartTime).append(" ms. "); + } else { + long spentSearching = renderStartTime - startTime; + msgbuf.append("Processing time: ").append(spentSearching).append(" ms. "); + } + + msgbuf.append("Result rendering/transfer: "); + msgbuf.append(commitStartTime - renderStartTime).append(" ms. "); + msgbuf.append("End transaction: "); + msgbuf.append(endTime - commitStartTime).append(" ms. "); + msgbuf.append("Total: ").append(totalTime).append(" ms. "); + msgbuf.append("Timeout: ").append(timeoutInterval).append(" ms. "); + msgbuf.append("Request string: ").append(req); + + log.log(LogLevel.WARNING, "Slow execution. " + msgbuf); + } + + private static class NullResponse extends ExtendedResponse { + NullResponse(int status) { + super(status); + } + + @Override + public void render(OutputStream output, ContentChannel networkChannel, + CompletionHandler handler) throws IOException { + // NOP + } + } + + private class LoggingHandler implements LoggingCompletionHandler { + + private final long startTime; + private final long renderStartTime; + private long commitStartTime; + private final HttpRequest httpRequest; + private final HttpResponse httpResponse; + private final ContentChannelOutputStream rendererWiring; + private final ExtendedResponse extendedResponse; + + LoggingHandler(long startTime, long renderStartTime, + HttpRequest httpRequest, HttpResponse httpResponse, + ContentChannelOutputStream rendererWiring) { + this.startTime = startTime; + this.renderStartTime = renderStartTime; + this.commitStartTime = renderStartTime; + this.httpRequest = httpRequest; + this.httpResponse = httpResponse; + this.rendererWiring = rendererWiring; + this.extendedResponse = actualOrNullObject(httpResponse); + } + + /** Set the commit start time to the current time */ + @Override + public void markCommitStart() { + this.commitStartTime = System.currentTimeMillis(); + } + + private ExtendedResponse actualOrNullObject(HttpResponse response) { + if (response instanceof ExtendedResponse) { + return (ExtendedResponse) response; + } else { + return new NullResponse(Response.Status.OK); + } + } + + @Override + public void completed() { + long endTime = System.currentTimeMillis(); + writeToLogs(endTime); + } + + @Override + public void failed(Throwable throwable) { + long endTime = System.currentTimeMillis(); + writeToLogs(endTime); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Got exception when writing to client: " + Exceptions.toMessageString(throwable)); + } + } + + private void writeToLogs(long endTime) { + final com.yahoo.jdisc.http.HttpRequest jdiscRequest = httpRequest.getJDiscRequest(); + + logTimes( + startTime, + getClientIP(jdiscRequest), + renderStartTime, + commitStartTime, + endTime, + jdiscRequest.getUri().toString(), + extendedResponse.getParsedQuery(), + extendedResponse.getTiming()); + + final Optional<AccessLogEntry> jdiscRequestAccessLogEntry + = AccessLoggingRequestHandler.getAccessLogEntry(jdiscRequest); + + if (jdiscRequestAccessLogEntry.isPresent()) { + // This means we are running with Jetty, not Netty. + // Actual logging will be done by the Jetty integration; here, we just need to populate. + httpResponse.populateAccessLogEntry(jdiscRequestAccessLogEntry.get()); + return; + } + + // We are running without Jetty. No access logging will be done at container level, so we do it here. + // TODO: Remove when netty support is removed. + + AccessLogEntry accessLogEntry = new AccessLogEntry(); + + populateAccessLogEntryNotCreatedByHttpServer( + accessLogEntry, + jdiscRequest, + extendedResponse.getTiming(), + httpRequest.getUri().toString(), + commitStartTime, + startTime, + rendererWiring.written(), + httpResponse.getStatus()); + httpResponse.populateAccessLogEntry(accessLogEntry); + + accessLog.log(accessLogEntry); + } + } + + private void populateAccessLogEntryNotCreatedByHttpServer( + final AccessLogEntry logEntry, + final com.yahoo.jdisc.http.HttpRequest httpRequest, + final Timing timing, + final String fullRequest, + final long commitStartTime, + final long startTime, + final long written, + final int status) { + try { + final InetSocketAddress remoteAddress = AccessLogUtil.getRemoteAddress(httpRequest); + final long evalStartTime = getEvalStart(timing, startTime); + logEntry.setIpV4Address(remoteHostAddress(httpRequest)); + logEntry.setTimeStamp(evalStartTime); + logEntry.setDurationBetweenRequestResponse(commitStartTime - evalStartTime); + logEntry.setReturnedContentSize(written); + logEntry.setStatusCode(status); + if (remoteAddress != null) { + logEntry.setRemoteAddress(remoteAddress); + logEntry.setRemotePort(remoteAddress.getPort()); + } + logEntry.setURI(AccessLogUtil.getUri(httpRequest)); + logEntry.setUserAgent(AccessLogUtil.getUserAgentHeader(httpRequest)); + logEntry.setReferer(AccessLogUtil.getReferrerHeader(httpRequest)); + logEntry.setHttpMethod(AccessLogUtil.getHttpMethod(httpRequest)); + logEntry.setHttpVersion(AccessLogUtil.getHttpVersion(httpRequest)); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Could not populate the access log [" + + fullRequest + "]", e); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java b/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java new file mode 100644 index 00000000000..47a6fd048f0 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.jdisc.application.MetricConsumer; + +/** + * <p>This is the interface to implement if one wishes to configure a non-default <tt>MetricConsumer</tt>. Simply + * add the implementing class as a component in your services.xml file.</p> + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public interface MetricConsumerFactory { + + public MetricConsumer newInstance(); +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java new file mode 100644 index 00000000000..76d8ac4f85f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.google.common.annotations.Beta; +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.application.ContainerBuilder; +import com.yahoo.jdisc.handler.BufferedContentChannel; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ReadableContentChannel; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.jdisc.test.TestDriver; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * A helper for making tests creating jDisc requests and checking their responses. + * + * @author bratseth + * @since 5.21 + */ +@Beta +public class RequestHandlerTestDriver implements AutoCloseable { + + private TestDriver driver; + + private MockResponseHandler responseHandler = null; + + /** Creates this with a binding to "http://localhost/*" */ + public RequestHandlerTestDriver(RequestHandler handler) { + this("http://localhost/*", handler); + } + + public RequestHandlerTestDriver(String binding, RequestHandler handler) { + driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(); + ContainerBuilder builder = driver.newContainerBuilder(); + builder.serverBindings().bind(binding, handler); + driver.activateContainer(builder); + } + + @Override + public void close() { + if (responseHandler != null) + responseHandler.readAll(); + assertTrue("Driver closed", driver.close()); + } + + /** Returns the jDisc level driver wrapped by this */ + public TestDriver jDiscDriver() { return driver; } + + /** Send a GET request */ + public MockResponseHandler sendRequest(String uri) { + return sendRequest(uri, HttpRequest.Method.GET); + } + + public MockResponseHandler sendRequest(String uri, HttpRequest.Method method) { + return sendRequest(uri, method, ""); + } + + public MockResponseHandler sendRequest(String uri, HttpRequest.Method method, String body) { + return sendRequest(uri, method, ByteBuffer.wrap(body.getBytes(StandardCharsets.UTF_8))); + } + + public MockResponseHandler sendRequest(String uri, HttpRequest.Method method, ByteBuffer body) { + responseHandler = new MockResponseHandler(); + Request request = HttpRequest.newServerRequest(driver, URI.create(uri), method); + request.context().put("contextVariable", 37); // TODO: Add a method for accepting a Request instead + ContentChannel requestContent = request.connect(responseHandler); + requestContent.write(body, null); + requestContent.close(null); + request.release(); + return responseHandler; + } + + /** Replaces all occurrences of 0-9 digits by d's */ + public String censorDigits(String s) { + return s.replaceAll("[0-9]","d"); + } + + /** Junit asserts are not available in the runtime dependencies */ + private static void assertTrue(String assertionMessage, boolean expectedTrue) { + if ( ! expectedTrue) + throw new RuntimeException("Assertion in ProcessingTestDriver failed: " + assertionMessage); + } + + public static class MockResponseHandler implements ResponseHandler { + + private final CountDownLatch latch = new CountDownLatch(1); + private final ReadableContentChannel content = new ReadableContentChannel(); + private final BufferedContentChannel buffer = new BufferedContentChannel(); + Response response = null; + + /** Blocks until there's a response (max 60 seconds). Returns this for chaining convenience */ + public MockResponseHandler awaitResponse() throws InterruptedException { + assertTrue("Handler responded", latch.await(60, TimeUnit.SECONDS)); + return this; + } + + /** + * Read the next piece of data from this channel even it blocking is needed. + * If all data is already read, this returns null. + */ + public String read() { + ByteBuffer nextBuffer = content.read(); + if (nextBuffer == null) return null; // end of transmission + return Charset.forName("utf-8").decode(nextBuffer).toString(); + } + + /** Returns the number of bytes available in the handler right now */ + public int available() { + return content.available(); + } + + /** + * Reads all data that will ever be produced by the channel attached to this, blocking as necessary. + * Returns an empty string if there is no data. + */ + public String readAll() { + String next; + StringBuilder responseString = new StringBuilder(); + while (null != (next = read()) ) + responseString.append(next); + return responseString.toString(); + } + + /** Consumes all <i>currently</i> available data, or return "" if no data is available right now. Never blocks. */ + public String readIfAvailable() { + StringBuilder b = new StringBuilder(); + while (content.available()>0) { + ByteBuffer nextBuffer = content.read(); + b.append(Charset.forName("utf-8").decode(nextBuffer).toString()); + } + return b.toString(); + } + + @Override + public ContentChannel handleResponse(Response response) { + this.response = response; + latch.countDown(); + + buffer.connectTo(this.content); + return buffer; + } + + public void clientClose() { + buffer.close(null); + } + + /** Returns the status code. Throws an exception if handleResponse is not called prior to calling this */ + public int getStatus() { + return response.getStatus(); + } + + public Response getResponse() { return response; } + + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java new file mode 100644 index 00000000000..e23104e484f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -0,0 +1,259 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.google.inject.Inject; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.handler.BufferedContentChannel; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.UnsafeContentInputStream; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.log.LogLevel; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * A simple HTTP request handler, using the {@link HttpRequest} and + * {@link HttpResponse} classes. Users need to override the + * {@link #handle(HttpRequest)} method in this class and the + * {@link HttpResponse#render(java.io.OutputStream)} method. + * + * @author musum + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author bratseth + * @since 5.1 + */ +public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler { + + public static final String CONTENT_TYPE = "Content-Type"; + public static final String DATE = "Date"; + private static final String RENDERING_ERRORS = "rendering_errors"; + + /** + * Logger for subclasses. + */ + protected final Logger log; + + + public ThreadedHttpRequestHandler(Executor executor) { + this(executor, null); + } + + @Inject + public ThreadedHttpRequestHandler(Executor executor, Metric metric) { + this(executor, metric, false); + } + + public ThreadedHttpRequestHandler(Executor executor, Metric metric, boolean allowAsyncResponse) { + super(executor, metric, allowAsyncResponse); + log = Logger.getLogger(this.getClass().getName()); + } + + /** + * Override this to implement a synchronous style handler. + * + * @param request incoming HTTP request + * @return a valid HTTP response for presentation to the user + */ + public abstract HttpResponse handle(HttpRequest request); + + /** + * Override this rather than handle(request) to be able to write to the channel before returning from this method. + * This default implementation calls handle(request) + */ + public HttpResponse handle(HttpRequest request, ContentChannel channel) { + return handle(request); + } + + @Override + public final void handleRequest(Request request, BufferedContentChannel requestContent, ResponseHandler responseHandler) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "In " + this.getClass() + ".handleRequest()"); + } + com.yahoo.jdisc.http.HttpRequest jdiscRequest = asHttpRequest(request); + HttpRequest httpRequest = new HttpRequest(jdiscRequest, new UnsafeContentInputStream(requestContent.toReadable())); + LazyContentChannel channel = null; + try { + channel = new LazyContentChannel(httpRequest, responseHandler, metric, log); + HttpResponse httpResponse = handle(httpRequest, channel); + channel.setHttpResponse(httpResponse); // may or may not have already been done + render(httpRequest, httpResponse, channel, jdiscRequest.creationTime(TimeUnit.MILLISECONDS)); + } catch (Exception e) { + metric.add(RENDERING_ERRORS, 1, null); + log.log(LogLevel.ERROR, "Uncaught exception handling request", e); + if (channel != null) { + channel.setHttpResponse(null); + channel.close(null); + } + } catch (Error e) { + // To make absolutely sure the VM exits on Error. + com.yahoo.protect.Process.logAndDie("java.lang.Error handling request", e); + } + } + + /** Render and return whether the channel was closed */ + private void render(HttpRequest request, HttpResponse httpResponse, + LazyContentChannel channel, long startTime) throws IOException { + LoggingCompletionHandler logOnCompletion = null; + ContentChannelOutputStream output = null; + try { + output = new ContentChannelOutputStream(channel); + logOnCompletion = createLoggingCompletionHandler(startTime, System.currentTimeMillis(), + httpResponse, request, output); + + addResponseHeaders(httpResponse, startTime); + + if (httpResponse instanceof AsyncHttpResponse) { + ((AsyncHttpResponse) httpResponse).render(output, channel, logOnCompletion); + } else { + httpResponse.render(output); + if (logOnCompletion != null) + logOnCompletion.markCommitStart(); + output.flush(); + } + } + catch (IOException e) { + metric.add(RENDERING_ERRORS, 1, null); + long time = System.currentTimeMillis() - startTime; + log.log(time < 900 ? LogLevel.INFO : LogLevel.WARNING, + "IO error while responding to " + " [" + + request.getUri() + "] " + "(total time " + + time + " ms) ", e); + try { if (output != null) output.flush(); } catch (Exception ignored) { } // TODO: Shouldn't this be channel.close()? + } finally { + if (channel != null && !(httpResponse instanceof AsyncHttpResponse)) { + channel.close(logOnCompletion); + } + } + } + + /** + * A content channel which will return the header and create the proper channel the first time content data needs + * to be written to it. + */ + public static class LazyContentChannel implements ContentChannel { + + /** The lazily created channel this wraps */ + private ContentChannel channel = null; + private boolean closed = false; + + // Fields needed to lazily create or close the channel */ + private HttpRequest httpRequest; + private HttpResponse httpResponse; + private final ResponseHandler responseHandler; + private final Metric metric; + private final Logger log; + + public LazyContentChannel(HttpRequest httpRequest, ResponseHandler responseHandler, Metric metric, Logger log) { + this.httpRequest = httpRequest; + this.responseHandler = responseHandler; + this.metric = metric; + this.log = log; + } + + /** This must be called before writing to this */ + public void setHttpResponse(HttpResponse httpResponse) { + if (httpResponse == null && this.httpResponse == null) // the handler in use returned a null response + httpResponse = new EmptyResponse(500); + this.httpResponse = httpResponse; + } + + @Override + public void write(ByteBuffer byteBuffer, CompletionHandler completionHandler) { + if (channel == null) + channel = handleResponse(); + channel.write(byteBuffer, completionHandler); + } + + @Override + public void close(CompletionHandler completionHandler) { + if ( closed ) return; + try { httpRequest.getData().close(); } catch (IOException e) {}; + if (channel == null) + channel = handleResponse(); + try { + channel.close(completionHandler); + } + catch (IllegalStateException e) { + // Ignore: Known to be thrown when the other party closes + } + closed = true; + } + + private ContentChannel handleResponse() { + try { + if (httpResponse == null) + throw new NullPointerException("Writing to a lazy content channel without calling setHttpResponse first"); + httpResponse.complete(); + return responseHandler.handleResponse(httpResponse.getJdiscResponse()); + } catch (Exception e) { + metric.add(RENDERING_ERRORS, 1, null); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Error writing response to client - connection probably terminated " + + "from client side.", e); + } + return new DevNullChannel(); // Ignore further operations on this + } + } + + private static class DevNullChannel implements ContentChannel { + + @Override + public void write(ByteBuffer byteBuffer, CompletionHandler completionHandler) { } + + @Override + public void close(CompletionHandler completionHandler) { } + + } + + } + + private void addResponseHeaders(HttpResponse httpResponse, long startTime) { + if ( ! httpResponse.headers().containsKey(CONTENT_TYPE) && httpResponse.getContentType() != null) { + StringBuilder s = new StringBuilder(httpResponse.getContentType()); + if (httpResponse.getCharacterEncoding() != null) { + s.append("; charset=").append(httpResponse.getCharacterEncoding()); + } + httpResponse.headers().put(CONTENT_TYPE, s.toString()); + } + addDateHeader(httpResponse, startTime); + } + + // Can be overridden to add Date HTTP response header. See bugs 3729021 and 6160137. + protected void addDateHeader(HttpResponse httpResponse, long startTime) { + } + + /** + * Override this to implement custom access logging. + * + * @param startTime + * execution start + * @param renderStartTime + * start of output rendering + * @param response + * the response which the log entry regards + * @param httpRequest + * the incoming HTTP request + * @param rendererWiring + * the stream the rendered response is written to, used for + * fetching length of rendered response + */ + protected LoggingCompletionHandler createLoggingCompletionHandler( + long startTime, long renderStartTime, HttpResponse response, + HttpRequest httpRequest, ContentChannelOutputStream rendererWiring) { + return null; + } + + protected com.yahoo.jdisc.http.HttpRequest asHttpRequest(Request request) { + if (!(request instanceof com.yahoo.jdisc.http.HttpRequest)) { + throw new IllegalArgumentException("Expected " + + com.yahoo.jdisc.http.HttpRequest.class.getName() + ", got " + request.getClass().getName()); + } + return (com.yahoo.jdisc.http.HttpRequest) request; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java new file mode 100644 index 00000000000..d54177a91e8 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -0,0 +1,228 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.google.inject.Inject; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.ResourceReference; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.BufferedContentChannel; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.OverloadException; +import com.yahoo.jdisc.handler.ReadableContentChannel; +import com.yahoo.jdisc.handler.ResponseDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.log.LogLevel; + +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.concurrent.GuardedBy; + +/** + * A request handler which assigns a worker thread to handle each request. + * This is mean to be subclasses by handlers who does work by executing each + * request in a separate thread. + * <p> + * Note that this means that subclass handlers are synchronous - the request io can + * continue after completion of the worker thread. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class ThreadedRequestHandler extends AbstractRequestHandler { + + private static final Logger log = Logger.getLogger(ThreadedRequestHandler.class.getName()); + private static final int TIMEOUT = Integer.parseInt(System.getProperty("ThreadedRequestHandler.timeout", "300")); + private final Executor executor; + protected final Metric metric; + private final boolean allowAsyncResponse; + + private static final Object rejectedExecutionsLock = new Object(); + @GuardedBy("rejectedExecutionsLock") + private static volatile int numRejectedRequests = 0; + @GuardedBy("rejectedExecutionsLock") + private static long currentFailureIntervalStartedMillis = 0L; + + protected ThreadedRequestHandler(Executor executor) { + this(executor, new NullRequestMetric()); + } + + @Inject + protected ThreadedRequestHandler(Executor executor, Metric metric) { + this(executor, metric, false); + } + + /** + * Creates a threaded request handler + * + * @param executor the executor to use to execute requests + * @param metric the metric object to which event in this are to be collected + * @param allowAsyncResponse true to allow the response header to be created asynchronously. + * If false (default), this will create an error response if the response header + * is not returned by the subclass of this before handleRequest returns. + */ + @Inject + protected ThreadedRequestHandler(Executor executor, Metric metric, boolean allowAsyncResponse) { + executor.getClass(); // throws NullPointerException + this.executor = executor; + this.metric = (metric == null) ? new NullRequestMetric() : metric; + this.allowAsyncResponse = allowAsyncResponse; + } + + /** + * Handles a request by assigning a worker thread to it. + * + * @throws OverloadException if thread pool has no available thread + */ + @Override + public final ContentChannel handleRequest(Request request, ResponseHandler responseHandler) { + request.setTimeout(TIMEOUT, TimeUnit.SECONDS); + BufferedContentChannel content = new BufferedContentChannel(); + final RequestTask command = new RequestTask(request, content, responseHandler); + try { + executor.execute(command); + } catch (RejectedExecutionException e) { + command.failOnOverload(); + throw new OverloadException("No available threads for " + getClass().getSimpleName(), e); + } finally { + logRejectedRequests(); + } + return content; + } + + private void logRejectedRequests() { + if (numRejectedRequests == 0) { + return; + } + final int numRejectedRequestsSnapshot; + synchronized (rejectedExecutionsLock) { + if (System.currentTimeMillis() - currentFailureIntervalStartedMillis < 1000) + return; + + numRejectedRequestsSnapshot = numRejectedRequests; + currentFailureIntervalStartedMillis = 0L; + numRejectedRequests = 0; + } + log.log(LogLevel.WARNING, "Rejected " + numRejectedRequestsSnapshot + " requests on cause of no available worker threads."); + } + + private void incrementRejectedRequests() { + synchronized (rejectedExecutionsLock) { + if (numRejectedRequests == 0) { + currentFailureIntervalStartedMillis = System.currentTimeMillis(); + } + numRejectedRequests += 1; + } + } + + protected abstract void handleRequest(Request request, BufferedContentChannel requestContent, + ResponseHandler responseHandler); + + private class RequestTask implements ResponseHandler, Runnable { + + final Request request; + private final ResourceReference requestReference; + final BufferedContentChannel content; + final ResponseHandler responseHandler; + private boolean hasResponded = false; + + RequestTask(Request request, BufferedContentChannel content, ResponseHandler responseHandler) { + this.request = request; + this.requestReference = request.refer(); + this.content = content; + this.responseHandler = responseHandler; + } + + @Override + public void run() { + try (ResourceReference reference = requestReference) { + processRequest(); + } + } + + private void processRequest() { + try { + ThreadedRequestHandler.this.handleRequest(request, content, this); + } catch (Exception e) { + log.log(Level.WARNING, "Uncaught exception in " + ThreadedRequestHandler.this.getClass().getName() + + ".", e); + } + consumeRequestContent(); + + // Unless the response is generated asynchronously, it should be generated before getting here, + // so respond with status 500 if we get here and no response has been generated. + if ( ! allowAsyncResponse) + respondWithErrorIfNotResponded(); + } + + @Override + public ContentChannel handleResponse(Response response) { + if ( tryHasResponded()) throw new IllegalStateException("Response already handled"); + return responseHandler.handleResponse(response); + } + + private boolean tryHasResponded() { + synchronized (this) { + if (hasResponded) return true; + hasResponded = true; + } + return false; + } + + private void respondWithErrorIfNotResponded() { + if ( tryHasResponded() ) return; + ResponseDispatch.newInstance(Response.Status.INTERNAL_SERVER_ERROR).dispatch(responseHandler); + log.warning("This handler is not async but did not produce a response. Responding with status 500." + + "(If this handler is async, pass a boolean true in the super constructor to avoid this.)"); + } + + private void consumeRequestContent() { + if (content.isConnected()) return; + ReadableContentChannel requestContent = new ReadableContentChannel(); + try { + content.connectTo(requestContent); + } catch (IllegalStateException e) { + return; + } + while (requestContent.read() != null) { + // consume all ignored content + } + } + + /** + * Clean up when the task can not be executed because no worker thread is available. + */ + public void failOnOverload() { + try (ResourceReference reference = requestReference) { + incrementRejectedRequests(); + logRejectedRequests(); + ResponseDispatch.newInstance(Response.Status.SERVICE_UNAVAILABLE).dispatch(responseHandler); + } + } + } + + private static class NullRequestMetric implements Metric { + @Override + public void set(String key, Number val, Context ctx) { + } + + @Override + public void add(String key, Number val, Context ctx) { + } + + @Override + public Context createContext(Map<String, ?> properties) { + return NullFeedContext.INSTANCE; + } + + private static class NullFeedContext implements Context { + private static final NullFeedContext INSTANCE = new NullFeedContext(); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java new file mode 100644 index 00000000000..8959114adea --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java @@ -0,0 +1,244 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + + +import static com.yahoo.container.protect.Error.BACKEND_COMMUNICATION_ERROR; +import static com.yahoo.container.protect.Error.BAD_REQUEST; +import static com.yahoo.container.protect.Error.FORBIDDEN; +import static com.yahoo.container.protect.Error.ILLEGAL_QUERY; +import static com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; +import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; +import static com.yahoo.container.protect.Error.NOT_FOUND; +import static com.yahoo.container.protect.Error.NO_BACKENDS_IN_SERVICE; +import static com.yahoo.container.protect.Error.TIMEOUT; +import static com.yahoo.container.protect.Error.UNAUTHORIZED; + +import java.util.Iterator; + +import com.yahoo.collections.Tuple2; +import com.yahoo.container.handler.Coverage; +import com.yahoo.container.handler.Timing; +import com.yahoo.container.http.BenchmarkingHeaders; +import com.yahoo.container.logging.HitCounts; +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.jdisc.Response; +import com.yahoo.processing.request.ErrorMessage; + +/** + * Static helper methods which implement the mapping between the ErrorMessage + * API and HTTP headers and return codes. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author bratseth + */ +public final class VespaHeaders { + + // response not directly supported by JDisc core + private static final int GATEWAY_TIMEOUT = 504; + private static final int BAD_GATEWAY = 502; + private static final int PRECONDITION_REQUIRED = 428; + private static final int TOO_MANY_REQUESTS = 429; + private static final int REQUEST_HEADER_FIELDS_TOO_LARGE = 431; + private static final int NETWORK_AUTHENTICATION_REQUIRED = 511; + + private static final Tuple2<Boolean, Integer> NO_MATCH = new Tuple2<>(false, Response.Status.OK); + + public static boolean benchmarkCoverage(boolean benchmarkOutput, HeaderFields headers) { + if (benchmarkOutput && headers.get(BenchmarkingHeaders.REQUEST_COVERAGE) != null) { + return true; + } else { + return false; + } + } + + public static boolean benchmarkOutput(com.yahoo.container.jdisc.HttpRequest request) { + return request.getHeader(BenchmarkingHeaders.REQUEST) != null; + } + + /** + * Add search benchmark output to the HTTP getHeaders + * + * @param responseHeaders The response to write the headers to. + * @param benchmarkCoverage True to include coverage headers. + * @param t The Timing to read data from. + * @param c The Counts to read data from. + * @param errorCount The error count. + * @param coverage The Coverage to read data from. + */ + public static void benchmarkOutput(HeaderFields responseHeaders, boolean benchmarkCoverage, + Timing t, HitCounts c, int errorCount, Coverage coverage) + { + final long renderStartTime = System.currentTimeMillis(); + if (c != null) { + // Fill inn response getHeaders + responseHeaders.add(BenchmarkingHeaders.NUM_HITS, String.valueOf(c.getRetrievedHitCount())); + responseHeaders.add(BenchmarkingHeaders.NUM_FASTHITS, String.valueOf(c.getSummaryCount())); + responseHeaders.add(BenchmarkingHeaders.TOTAL_HIT_COUNT, String.valueOf(c.getTotalHitCount())); + responseHeaders.add(BenchmarkingHeaders.QUERY_HITS, String.valueOf(c.getRequestedHits())); + responseHeaders.add(BenchmarkingHeaders.QUERY_OFFSET, String.valueOf(c.getRequestedOffset())); + } + responseHeaders.add(BenchmarkingHeaders.NUM_ERRORS, String.valueOf(errorCount)); + if (t != null) { + if (t.getSummaryStartTime() != 0) { + responseHeaders.add(BenchmarkingHeaders.SEARCH_TIME, + String.valueOf(t.getSummaryStartTime() - t.getQueryStartTime())); + responseHeaders.add(BenchmarkingHeaders.ATTR_TIME, "0"); + responseHeaders.add(BenchmarkingHeaders.FILL_TIME, + String.valueOf(renderStartTime - t.getSummaryStartTime())); + } else { + responseHeaders.add(BenchmarkingHeaders.SEARCH_TIME, + String.valueOf(renderStartTime - t.getQueryStartTime())); + responseHeaders.add(BenchmarkingHeaders.ATTR_TIME, "0"); + responseHeaders.add(BenchmarkingHeaders.FILL_TIME, "0"); + } + } + + if (benchmarkCoverage && coverage != null) { + responseHeaders.add(BenchmarkingHeaders.DOCS_SEARCHED, String.valueOf(coverage.getDocs())); + responseHeaders.add(BenchmarkingHeaders.NODES_SEARCHED, String.valueOf(coverage.getNodes())); + responseHeaders.add(BenchmarkingHeaders.FULL_COVERAGE, String.valueOf(coverage.getFull() ? 1 : 0)); + } + } + + /** + * (during normal execution) return 200 unless this is not a success or a 4xx error is requested. + * + * @param isSuccess Whether or not the response represents a success. + * @param mainError The main error of the response, if any. + * @param allErrors All the errors of the response, if any. + * @return The status code of the given response. + */ + public static int getStatus(boolean isSuccess, ErrorMessage mainError, Iterator<? extends ErrorMessage> allErrors) { + // Do note, SearchResponse has its own implementation of isSuccess() + if (isSuccess) { + Tuple2<Boolean, Integer> status = webServiceCodes(mainError, allErrors); + if (status.first) { + return status.second; + } else { + return Response.Status.OK; + } + } + return getEagerErrorStatus(mainError, allErrors); + } + + private static Tuple2<Boolean, Integer> webServiceCodes(ErrorMessage mainError, Iterator<? extends ErrorMessage> allErrors) { + if (mainError == null) return NO_MATCH; + + Iterator<? extends ErrorMessage> errorIterator = allErrors; + if (errorIterator != null && errorIterator.hasNext()) { + for (; errorIterator.hasNext();) { + ErrorMessage error = errorIterator.next(); + Tuple2<Boolean, Integer> status = chooseWebServiceStatus(error); + if (status.first) { + return status; + } + } + } else { + Tuple2<Boolean, Integer> status = chooseWebServiceStatus(mainError); + if (status.first) { + return status; + } + } + return NO_MATCH; + } + + + private static Tuple2<Boolean, Integer> chooseWebServiceStatus(ErrorMessage error) { + if (isHttpStatusCode(error.getCode())) + return new Tuple2<>(true, error.getCode()); + if (error.getCode() == FORBIDDEN.code) + return new Tuple2<>(true, Response.Status.FORBIDDEN); + if (error.getCode() == UNAUTHORIZED.code) + return new Tuple2<>(true, Response.Status.UNAUTHORIZED); + if (error.getCode() == NOT_FOUND.code) + return new Tuple2<>(true, Response.Status.NOT_FOUND); + if (error.getCode() == BAD_REQUEST.code) + return new Tuple2<>(true, Response.Status.BAD_REQUEST); + if (error.getCode() == INTERNAL_SERVER_ERROR.code) + return new Tuple2<>(true, Response.Status.INTERNAL_SERVER_ERROR); + return NO_MATCH; + } + + // TODO: The status codes in jDisc should be an ENUM so we can enumerate the values + private static boolean isHttpStatusCode(int code) { + switch (code) { + case Response.Status.OK : + case Response.Status.MOVED_PERMANENTLY : + case Response.Status.FOUND : + case Response.Status.TEMPORARY_REDIRECT : + case Response.Status.BAD_REQUEST : + case Response.Status.UNAUTHORIZED : + case Response.Status.FORBIDDEN : + case Response.Status.NOT_FOUND : + case Response.Status.METHOD_NOT_ALLOWED : + case Response.Status.NOT_ACCEPTABLE : + case Response.Status.REQUEST_TIMEOUT : + case Response.Status.INTERNAL_SERVER_ERROR : + case Response.Status.NOT_IMPLEMENTED : + case Response.Status.SERVICE_UNAVAILABLE : + case Response.Status.VERSION_NOT_SUPPORTED : + case GATEWAY_TIMEOUT : + case BAD_GATEWAY : + case PRECONDITION_REQUIRED : + case TOO_MANY_REQUESTS : + case REQUEST_HEADER_FIELDS_TOO_LARGE : + case NETWORK_AUTHENTICATION_REQUIRED : + return true; + default: + return false; + } + } + + + /** + * Returns 5xx or 4xx if there is any error present in the result, 200 otherwise + * + * @param mainError The main error of the response. + * @param allErrors All the errors of the response, if any. + * @return The error status code of the given response. + */ + public static int getEagerErrorStatus(ErrorMessage mainError, Iterator<? extends ErrorMessage> allErrors) { + if (mainError == null ) return Response.Status.OK; + + // Iterate over all errors + if (allErrors != null && allErrors.hasNext()) { + for (; allErrors.hasNext();) { + ErrorMessage error = allErrors.next(); + Tuple2<Boolean, Integer> status = chooseStatusFromError(error); + if (status.first) { + return status.second; + } + } + } else { + Tuple2<Boolean, Integer> status = chooseStatusFromError(mainError); + if (status.first) { + return status.second; + } + } + + // Default return code for errors + return Response.Status.INTERNAL_SERVER_ERROR; + } + + private static Tuple2<Boolean, Integer> chooseStatusFromError(ErrorMessage error) { + + Tuple2<Boolean, Integer> webServiceStatus = chooseWebServiceStatus(error); + if (webServiceStatus.first) { + return webServiceStatus; + } + if (error.getCode() == NO_BACKENDS_IN_SERVICE.code) + return new Tuple2<>(true, Response.Status.SERVICE_UNAVAILABLE); + if (error.getCode() == TIMEOUT.code) + return new Tuple2<>(true, Response.Status.GATEWAY_TIMEOUT); + if (error.getCode() == BACKEND_COMMUNICATION_ERROR.code) + return new Tuple2<>(true, Response.Status.SERVICE_UNAVAILABLE); + if (error.getCode() == ILLEGAL_QUERY.code) + return new Tuple2<>(true, Response.Status.BAD_REQUEST); + if (error.getCode() == INVALID_QUERY_PARAMETER.code) + return new Tuple2<>(true, Response.Status.BAD_REQUEST); + return NO_MATCH; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/package-info.java b/container-core/src/main/java/com/yahoo/container/jdisc/package-info.java new file mode 100644 index 00000000000..3eb9b19bb76 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.container.jdisc; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/CountMetric.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/CountMetric.java new file mode 100644 index 00000000000..a4b6506adb7 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/CountMetric.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +/** + * A metric which is counting an accumulative value + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public final class CountMetric extends MetricValue { + + private long count; + + private CountMetric(long count) { + this.count = count; + } + + @Override + void add(Number num) { + this.count += num.longValue(); + } + + @Override + void add(MetricValue value) { + CountMetric rhs = (CountMetric)value; + this.count += rhs.count; + } + + /** Returns the accumulated count of this metric in the time interval */ + public long getCount() { + return count; + } + + public static CountMetric newSingleValue(Number val) { + long lval = val.longValue(); + return new CountMetric(lval); + } + + public static CountMetric newInstance(long val) { + return new CountMetric(val); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/GaugeMetric.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/GaugeMetric.java new file mode 100644 index 00000000000..da8c318f055 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/GaugeMetric.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import java.util.List; +import java.util.Optional; + +import com.yahoo.collections.Tuple2; + +/** + * A metric which contains a gauge value, i.e a value which represents the magnitude of something + * measured at a point in time. This metric value contains some additional information about the distribution + * of this gauge value in the time interval this metric is for. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public final class GaugeMetric extends MetricValue { + + private double last; + private double max; + private double min; + private double sum; + private long count; + private Optional<List<Tuple2<String, Double>>> percentiles; + + private GaugeMetric(double last, double max, double min, double sum, long count, Optional<List<Tuple2<String, Double>>> percentiles) { + this.last = last; + this.max = max; + this.min = min; + this.sum = sum; + this.count = count; + this.percentiles = percentiles; + } + + @Override + void add(Number val) { + double dval = val.doubleValue(); + last = dval; + if (dval > max) { + max = dval; + } + if (dval < min) { + min = dval; + } + sum += dval; + ++count; + } + + @Override + void add(MetricValue val) { + GaugeMetric rhs = (GaugeMetric)val; + last = rhs.last; + if (rhs.max > max) { + max = rhs.max; + } + if (rhs.min < min) { + min = rhs.min; + } + sum += rhs.sum; + count += rhs.count; + } + + /** + * Returns the average reading of this value in the time interval, or--if no + * value has been set within this period--the value of 'last' from the + * most recent interval where this metric was set. + */ + public double getAverage() { + return count != 0 ? (sum / count) : last; + } + + /** + * Returns the most recent assignment of this metric in the time interval, + * or--if no value has been set within this period--the value of 'last' + * from the most recent interval where this metric was set. + */ + public double getLast() { return last; } + + /** + * Returns the max value of this metric in the time interval, or--if no + * value has been set within this period--the value of 'last' from the + * most recent interval where this metric was set. + */ + public double getMax() { + return (count == 0) ? last : max; + } + + /** + * Returns the min value of this metric in the time interval, or--if no + * value has been set within this period--the value of 'last' from the + * most recent interval where this metric was set. + */ + public double getMin() { + return (count == 0) ? last : min; + } + + /** Returns the sum of all assignments of this metric in the time interval */ + public double getSum() { + return sum; + } + + /** Returns the number of assignments of this value in the time interval */ + public long getCount() { + return count; + } + + /** Returns the 95th percentile value for this time interval */ + public Optional<List<Tuple2<String, Double>>> getPercentiles() { + return percentiles; + } + + /** + * Create a partial clone of this gauge where the value of 'last' is + * carried over to the new gauge with all other fields left at defaults + * (0 for count and sum, biggest possible double for min, smallest possible + * double for max). Note that since count is 0, these extreme values will + * never be output from the min/max getters as these will return 'last' + * in this case. + * @return A new gauge instance + */ + public GaugeMetric newWithPreservedLastValue() { + // min/max set to enforce update of these values on first call to add() + return new GaugeMetric(last, Double.MIN_VALUE, Double.MAX_VALUE, 0, 0, Optional.empty()); + } + + public static GaugeMetric newSingleValue(Number val) { + double dval = val.doubleValue(); + return new GaugeMetric(dval, dval, dval, dval, 1, Optional.empty()); + } + + public static GaugeMetric newInstance(double last, double max, double min, double sum, long count) { + return new GaugeMetric(last, max, min, sum, count, Optional.empty()); + } + + public static GaugeMetric newInstance(double last, double max, double min, double sum, long count, Optional<List<Tuple2<String, Double>>> percentiles) { + return new GaugeMetric(last, max, min, sum, count, percentiles); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricDimensions.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricDimensions.java new file mode 100644 index 00000000000..d3a38e11ef0 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricDimensions.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import java.util.Map; + +/** + * A set of metric dimensions, which are key-value string pairs. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public interface MetricDimensions extends Iterable<Map.Entry<String, String>> { + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricSet.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricSet.java new file mode 100644 index 00000000000..fe24b55798b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricSet.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * A set of metrics. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public final class MetricSet implements Iterable<Map.Entry<String, MetricValue>> { + + private final static Logger log = Logger.getLogger(MetricSet.class.getName()); + + private final Map<String, MetricValue> data; + + MetricSet() { + data = new HashMap<>(); + } + + public MetricSet(Map<String, MetricValue> data) { + this.data = data; + } + + /** Returns all metrics in this */ + @Override + public Iterator<Map.Entry<String, MetricValue>> iterator() { + return data.entrySet().iterator(); + } + + /** Returns a metric value */ + public MetricValue get(String key) { + return data.get(key); + } + + void add(String key, Number val) { + add(key, CountMetric.newSingleValue(val)); + } + + void set(String key, Number val) { + add(key, GaugeMetric.newSingleValue(val)); + } + + void add(MetricSet metricSet) { + for (Map.Entry<String, MetricValue> entry : metricSet) { + add(entry.getKey(), entry.getValue()); + } + } + + private void add(String key, MetricValue value) { + MetricValue existingValue = data.get(key); + + if (existingValue == null) { + data.put(key, value); + return; + } + + if ( ! existingValue.getClass().isAssignableFrom(value.getClass())) { + log.info("Resetting metric '" + key + "' as it changed type. " + + "If you see this outside of deployment changes it means you incorrectly call both set() and add() " + + "on the same metric"); + data.put(key, value); + return; + } + + existingValue.add(value); + } + + boolean isEmpty() { return data.isEmpty(); } + + /** + * Create and return a MetricSet which carries over the last values + * set for gauges in the this MetricSet. Aggregate metrics are currently + * not carried over and will not be present in the returned set. + */ + public MetricSet partialClone() { + return new MetricSet( + data.entrySet().stream() + .filter(kv -> kv.getValue() instanceof GaugeMetric) + .collect(Collectors.toMap( + kv -> kv.getKey(), + kv -> ((GaugeMetric)kv.getValue()).newWithPreservedLastValue()))); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricSnapshot.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricSnapshot.java new file mode 100644 index 00000000000..6418f84e95b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricSnapshot.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * A snapshot of the metrics of this system in a particular time interval. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public final class MetricSnapshot implements Iterable<Map.Entry<MetricDimensions, MetricSet>> { + + private final Map<MetricDimensions, MetricSet> data; + private final long fromMillis; + private final long toMillis; + + public MetricSnapshot(long from, long to, TimeUnit unit, Map<MetricDimensions, MetricSet> data) { + this.fromMillis = unit.toMillis(from); + this.toMillis = unit.toMillis(to); + this.data = data; + } + + MetricSnapshot() { + this(0, 0, TimeUnit.MILLISECONDS, new HashMap<>()); + } + + MetricSnapshot(long from, long to, TimeUnit unit) { + this(from, to, unit, new HashMap<>()); + } + + private static MetricSnapshot createWithMetrics(Map<MetricDimensions, MetricSet> data) { + return new MetricSnapshot(0, 0, TimeUnit.MILLISECONDS, data); + } + + public long getFromTime(TimeUnit unit) { + return unit.convert(fromMillis, TimeUnit.MILLISECONDS); + } + + public long getToTime(TimeUnit unit) { + return unit.convert(toMillis, TimeUnit.MILLISECONDS); + } + + /** Returns all the metrics in this snapshot. */ + @Override + public Iterator<Map.Entry<MetricDimensions, MetricSet>> iterator() { + return data.entrySet().iterator(); + } + + void add(MetricDimensions dim, String key, Number val) { + metricSet(dim).add(key, val); + } + + void set(MetricDimensions dim, String key, Number val) { + metricSet(dim).set(key, val); + } + + void add(MetricSnapshot snapshot) { + for (Map.Entry<MetricDimensions, MetricSet> entry : snapshot) { + MetricSet metricSet = data.get(entry.getKey()); + if (metricSet != null) { + metricSet.add(entry.getValue()); + } else { + data.put(entry.getKey(), entry.getValue()); + } + } + } + + /** Returns a metric set from this snapshot for a given set of dimensions */ + public MetricSet metricSet(MetricDimensions dim) { + MetricSet metricSet = data.get(dim); + if (metricSet == null) { + data.put(dim, metricSet = new MetricSet()); + } + return metricSet; + } + + /** + * Create a new snapshot instance where Gauge metrics are preserved + * with the last-values they have in this snapshot instance. + */ + public MetricSnapshot createSnapshot() { + Map<MetricDimensions, MetricSet> newData = new HashMap<>(); + for (Map.Entry<MetricDimensions, MetricSet> entry : data.entrySet()) { + MetricSet newSet = entry.getValue().partialClone(); + if (!newSet.isEmpty()) { + newData.put(entry.getKey(), newSet); + } + } + return createWithMetrics(newData); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricValue.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricValue.java new file mode 100644 index 00000000000..2faefaf8654 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricValue.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +/** + * A metric value + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public abstract class MetricValue { + + abstract void add(Number val); + + abstract void add(MetricValue val); + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/SnapshotProvider.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/SnapshotProvider.java new file mode 100644 index 00000000000..b72ee43bd7c --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/SnapshotProvider.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import java.io.PrintStream; + +/** + * An interface for components supplying a state snapshot where persistence and + * other pre-processing has been done. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public interface SnapshotProvider { + public MetricSnapshot latestSnapshot(); + public void histogram(PrintStream output); +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java new file mode 100644 index 00000000000..011944c6a23 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java @@ -0,0 +1,389 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import com.google.common.base.Splitter; +import com.google.inject.Inject; +import com.yahoo.collections.Tuple2; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.core.ApplicationMetadataConfig; +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.Timer; +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.HttpHeaders; +import com.yahoo.metrics.MetricsPresentationConfig; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.io.PrintStream; +import java.io.ByteArrayOutputStream; + +/** + * A handler which returns state (health) information from this container instance: Status and metrics. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class StateHandler extends AbstractRequestHandler { + + private static final String METRICS_API_NAME = "metrics"; + private static final String HISTOGRAMS_API_NAME = "metrics/histograms"; + private static final String CONFIG_GENERATION_API_NAME = "config"; + private static final String HEALTH_API_NAME = "health"; + public static final String STATE_API_ROOT = "/state/v1"; + private final static MetricDimensions NULL_DIMENSIONS = StateMetricContext.newInstance(null); + private final StateMonitor monitor; + private final Timer timer; + private final byte[] config; + private final SnapshotProvider snapshotPreprocessor; + private static final Splitter onSlash = Splitter.on('/'); + + @Inject + public StateHandler(StateMonitor monitor, Timer timer, ApplicationMetadataConfig config, + ComponentRegistry<SnapshotProvider> preprocessors, MetricsPresentationConfig presentation) { + this.monitor = monitor; + this.timer = timer; + this.config = buildConfigOutput(config); + List<SnapshotProvider> allPreprocessors = preprocessors.allComponents(); + if (presentation.slidingwindow() && allPreprocessors.size() > 0) { + snapshotPreprocessor = allPreprocessors.get(0); + } else { + snapshotPreprocessor = null; + } + } + + @Override + public ContentChannel handleRequest(final Request request, ResponseHandler handler) { + new ResponseDispatch() { + + @Override + protected Response newResponse() { + Response response = new Response(Response.Status.OK); + response.headers().add(HttpHeaders.Names.CONTENT_TYPE, resolveContentType(request.getUri())); + return response; + } + + @Override + protected Iterable<ByteBuffer> responseContent() { + return Collections.singleton(buildContent(request.getUri())); + } + }.dispatch(handler); + return null; + } + + private String resolveContentType(URI requestUri) { + if (resolvePath(requestUri).equals(HISTOGRAMS_API_NAME)) { + return "text/plain; charset=utf-8"; + } else { + return "application/json"; + } + } + + private ByteBuffer buildContent(URI requestUri) { + String suffix = resolvePath(requestUri); + switch (suffix) { + case "": + return ByteBuffer.wrap(apiLinks(requestUri)); + case CONFIG_GENERATION_API_NAME: + return ByteBuffer.wrap(config); + case HISTOGRAMS_API_NAME: + return ByteBuffer.wrap(buildHistogramsOutput()); + case HEALTH_API_NAME: + case METRICS_API_NAME: + return ByteBuffer.wrap(buildMetricOutput(suffix)); + default: + // XXX should possibly do something else here + return ByteBuffer.wrap(buildMetricOutput(suffix)); + } + } + + private byte[] apiLinks(URI requestUri) { + try { + int port = requestUri.getPort(); + String host = requestUri.getHost(); + StringBuilder base = new StringBuilder("http://"); + base.append(host); + if (port != -1) { + base.append(":").append(port); + } + base.append(STATE_API_ROOT); + String uriBase = base.toString(); + JSONArray linkList = new JSONArray(); + for (String api : new String[] { METRICS_API_NAME, CONFIG_GENERATION_API_NAME, HEALTH_API_NAME}) { + JSONObject resource = new JSONObject(); + resource.put("url", uriBase + "/" + api); + linkList.put(resource); + } + return new JSONObjectWithLegibleException() + .put("resources", linkList) + .toString(4).getBytes(StandardCharsets.UTF_8); + } catch (JSONException e) { + throw new RuntimeException("Bad JSON construction.", e); + } + } + + private static String resolvePath(URI uri) { + String path = uri.getPath(); + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + if (path.startsWith(STATE_API_ROOT)) { + path = path.substring(STATE_API_ROOT.length()); + } + if (path.startsWith("/")) { + path = path.substring(1); + } + return path; + } + + private static byte[] buildConfigOutput(ApplicationMetadataConfig config) { + try { + return new JSONObjectWithLegibleException() + .put(CONFIG_GENERATION_API_NAME, new JSONObjectWithLegibleException() + .put("generation", config.generation()) + .put("container", new JSONObjectWithLegibleException() + .put("generation", config.generation()))) + .toString(4).getBytes(StandardCharsets.UTF_8); + } catch (JSONException e) { + throw new RuntimeException("Bad JSON construction.", e); + } + } + + private byte[] buildMetricOutput(String consumer) { + try { + return buildJsonForConsumer(consumer).toString(4).getBytes(StandardCharsets.UTF_8); + } catch (JSONException e) { + throw new RuntimeException("Bad JSON construction.", e); + } + } + + private byte[] buildHistogramsOutput() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + if (snapshotPreprocessor != null) { + snapshotPreprocessor.histogram(new PrintStream(baos)); + } + return baos.toByteArray(); + } + + private JSONObjectWithLegibleException buildJsonForConsumer(String consumer) throws JSONException { + JSONObjectWithLegibleException ret = new JSONObjectWithLegibleException(); + ret.put("time", timer.currentTimeMillis()); + ret.put("status", new JSONObjectWithLegibleException().put("code", "up")); + ret.put(METRICS_API_NAME, buildJsonForSnapshot(consumer, getSnapshot())); + return ret; + } + + private MetricSnapshot getSnapshot() { + if (snapshotPreprocessor == null) { + return monitor.snapshot(); + } else { + return snapshotPreprocessor.latestSnapshot(); + } + } + + private JSONObjectWithLegibleException buildJsonForSnapshot(String consumer, MetricSnapshot metricSnapshot) throws JSONException { + if (metricSnapshot == null) { + return new JSONObjectWithLegibleException(); + } + JSONObjectWithLegibleException jsonMetric = new JSONObjectWithLegibleException(); + jsonMetric.put("snapshot", new JSONObjectWithLegibleException() + .put("from", metricSnapshot.getFromTime(TimeUnit.MILLISECONDS) / 1000.0) + .put("to", metricSnapshot.getToTime(TimeUnit.MILLISECONDS) / 1000.0)); + + boolean includeDimensions = !consumer.equals(HEALTH_API_NAME); + long periodInMillis = metricSnapshot.getToTime(TimeUnit.MILLISECONDS) - + metricSnapshot.getFromTime(TimeUnit.MILLISECONDS); + for (Tuple tuple : collapseMetrics(metricSnapshot, consumer)) { + JSONObjectWithLegibleException jsonTuple = new JSONObjectWithLegibleException(); + jsonTuple.put("name", tuple.key); + if (tuple.val instanceof CountMetric) { + CountMetric count = (CountMetric)tuple.val; + jsonTuple.put("values", new JSONObjectWithLegibleException() + .put("count", count.getCount()) + .put("rate", (count.getCount() * 1000.0) / periodInMillis)); + } else if (tuple.val instanceof GaugeMetric) { + GaugeMetric gauge = (GaugeMetric) tuple.val; + JSONObjectWithLegibleException valueFields = new JSONObjectWithLegibleException(); + valueFields.put("average", gauge.getAverage()) + .put("count", gauge.getCount()) + .put("last", gauge.getLast()) + .put("max", gauge.getMax()) + .put("min", gauge.getMin()) + .put("rate", (gauge.getCount() * 1000.0) / periodInMillis); + if (gauge.getPercentiles().isPresent()) { + for (Tuple2<String, Double> prefixAndValue : gauge.getPercentiles().get()) { + valueFields.put(prefixAndValue.first + "percentile", prefixAndValue.second.doubleValue()); + } + } + jsonTuple.put("values", valueFields); + } else { + throw new UnsupportedOperationException(tuple.val.getClass().getName()); + } + Iterator<Map.Entry<String, String>> it = tuple.dim.iterator(); + if (it.hasNext() && includeDimensions) { + JSONObjectWithLegibleException jsonDim = new JSONObjectWithLegibleException(); + while (it.hasNext()) { + Map.Entry<String, String> entry = it.next(); + jsonDim.put(entry.getKey(), entry.getValue()); + } + jsonTuple.put("dimensions", jsonDim); + } + jsonMetric.append("values", jsonTuple); + } + return jsonMetric; + } + + private static List<Tuple> collapseMetrics(MetricSnapshot snapshot, String consumer) { + switch (consumer) { + case HEALTH_API_NAME: + return collapseHealthMetrics(snapshot); + case "all": // deprecated name + case METRICS_API_NAME: + return collapseAllMetrics(snapshot); + default: + throw new IllegalArgumentException("Unknown consumer '" + consumer + "'."); + } + } + + private static List<Tuple> collapseHealthMetrics(MetricSnapshot snapshot) { + Tuple requestsPerSecond = new Tuple(NULL_DIMENSIONS, "requestsPerSecond", null); + Tuple latencySeconds = new Tuple(NULL_DIMENSIONS, "latencySeconds", null); + for (Map.Entry<MetricDimensions, MetricSet> entry : snapshot) { + MetricSet metricSet = entry.getValue(); + MetricValue val = metricSet.get("serverTotalSuccessfulResponseLatency"); + if (val instanceof GaugeMetric) { + GaugeMetric gauge = (GaugeMetric)val; + latencySeconds.add(GaugeMetric.newInstance(gauge.getLast() / 1000, + gauge.getMax() / 1000, + gauge.getMin() / 1000, + gauge.getSum() / 1000, + gauge.getCount())); + } + requestsPerSecond.add(metricSet.get("serverNumSuccessfulResponses")); + } + List<Tuple> lst = new ArrayList<>(); + if (requestsPerSecond.val != null) { + lst.add(requestsPerSecond); + } + if (latencySeconds.val != null) { + lst.add(latencySeconds); + } + return lst; + } + + private static List<Tuple> collapseAllMetrics(MetricSnapshot snapshot) { + List<Tuple> lst = new ArrayList<>(); + for (Map.Entry<MetricDimensions, MetricSet> snapshotEntry : snapshot) { + for (Map.Entry<String, MetricValue> metricSetEntry : snapshotEntry.getValue()) { + lst.add(new Tuple(snapshotEntry.getKey(), metricSetEntry.getKey(), metricSetEntry.getValue())); + } + } + return lst; + } + + private static class Tuple { + + final MetricDimensions dim; + final String key; + MetricValue val; + + Tuple(MetricDimensions dim, String key, MetricValue val) { + this.dim = dim; + this.key = key; + this.val = val; + } + + void add(MetricValue val) { + if (val == null) { + return; + } + if (this.val == null) { + this.val = val; + } else { + this.val.add(val); + } + } + } + + static class JSONObjectWithLegibleException extends JSONObject { + @Override + public JSONObject put(String s, boolean b) { + try { + return super.put(s, b); + } catch (JSONException e) { + throw new RuntimeException(getErrorMessage(s, b, e), e); + } + } + + @Override + public JSONObject put(String s, double v) { + try { + Double guardedVal = (((Double)v).isNaN() || ((Double)v).isInfinite()) ? + 0.0 : v; + return super.put(s, guardedVal); + } catch (JSONException e) { + throw new RuntimeException(getErrorMessage(s, v, e), e); + } + } + + @Override + public JSONObject put(String s, int i) { + try { + return super.put(s, i); + } catch (JSONException e) { + throw new RuntimeException(getErrorMessage(s, i, e), e); + } + } + + @Override + public JSONObject put(String s, long l) { + try { + return super.put(s, l); + } catch (JSONException e) { + throw new RuntimeException(getErrorMessage(s, l, e), e); + } + } + + @Override + public JSONObject put(String s, Collection collection) { + try { + return super.put(s, collection); + } catch (JSONException e) { + throw new RuntimeException(getErrorMessage(s, collection, e), e); + } + } + + @Override + public JSONObject put(String s, Map map) { + try { + return super.put(s, map); + } catch (JSONException e) { + throw new RuntimeException(getErrorMessage(s, map, e), e); + } + } + + @Override + public JSONObject put(String s, Object o) { + try { + return super.put(s, o); + } catch (JSONException e) { + throw new RuntimeException(getErrorMessage(s, o, e), e); + } + } + + private String getErrorMessage(String key, Object value, JSONException e) { + return "Trying to add invalid JSON object with key '" + key + + "' and value '" + value + + "' - " + e.getMessage(); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricConsumer.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricConsumer.java new file mode 100644 index 00000000000..31453dfb85b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricConsumer.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.application.MetricConsumer; + +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +final class StateMetricConsumer implements MetricConsumer { + + final static Metric.Context NULL_CONTEXT = StateMetricContext.newInstance(null); + private final Object lock = new Object(); + private MetricSnapshot metricSnapshot = new MetricSnapshot(); + + @Override + public void set(String key, Number val, Metric.Context ctx) { + synchronized (lock) { + metricSnapshot.set(dimensionsOrDefault(ctx), key, val); + } + } + + private MetricDimensions dimensionsOrDefault(Metric.Context ctx) { + return (MetricDimensions)(ctx != null ? ctx : NULL_CONTEXT); + } + + @Override + public void add(String key, Number val, Metric.Context ctx) { + synchronized (lock) { + metricSnapshot.add(dimensionsOrDefault(ctx), key, val); + } + } + + @Override + public Metric.Context createContext(Map<String, ?> properties) { + return StateMetricContext.newInstance(properties); + } + + MetricSnapshot createSnapshot() { + MetricSnapshot metricSnapshot; + synchronized (lock) { + metricSnapshot = this.metricSnapshot; + this.metricSnapshot = this.metricSnapshot.createSnapshot(); + } + return metricSnapshot; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricContext.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricContext.java new file mode 100644 index 00000000000..d4568c0dea6 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricContext.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import com.yahoo.jdisc.Metric; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * A context implementation whose identity is the key and values such that this can be used as + * a key in metrics lookups. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public final class StateMetricContext implements MetricDimensions, Metric.Context { + + private final Map<String, String> data; + private final int hashCode; + + private StateMetricContext(Map<String, String> data) { + this.data = data; + this.hashCode = data.hashCode(); + } + + @Override + public Iterator<Map.Entry<String, String>> iterator() { + return data.entrySet().iterator(); + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public boolean equals(Object obj) { + return (obj == this) || + (obj instanceof StateMetricContext && ((StateMetricContext)obj).data.equals(data)); + } + + public static StateMetricContext newInstance(Map<String, ?> properties) { + Map<String, String> data; + if (properties != null) { + data = new HashMap<>(properties.size()); + for (Map.Entry<String, ?> entry : properties.entrySet()) { + data.put(entry.getKey(), entry.getValue() != null ? entry.getValue().toString() : null); + } + data = Collections.unmodifiableMap(data); + } else { + data = Collections.emptyMap(); + } + return new StateMetricContext(data); + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java new file mode 100644 index 00000000000..b81c758aa32 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.jdisc.config.HealthMonitorConfig; +import com.yahoo.jdisc.Timer; +import com.yahoo.jdisc.application.MetricConsumer; + +import java.util.Map; +import java.util.TreeSet; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * A statemonitor keeps track of the current metrics state of a container. + * It is used by jDisc to hand out metric update API endpoints to workers through {@link #newMetricConsumer}, + * and to inspect the current accumulated state of metrics through {@link #snapshot}. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class StateMonitor extends AbstractComponent { + + private final static Logger log = Logger.getLogger(StateMonitor.class.getName()); + private final CopyOnWriteArrayList<StateMetricConsumer> consumers = new CopyOnWriteArrayList<>(); + private final Thread thread; + private final Timer timer; + private final long snapshotIntervalMs; + private long lastSnapshotTimeMs; + private volatile MetricSnapshot snapshot; + private final TreeSet<String> valueNames = new TreeSet<>(); + + @Inject + public StateMonitor(HealthMonitorConfig config, Timer timer) { + this.timer = timer; + this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)); + this.lastSnapshotTimeMs = timer.currentTimeMillis(); + thread = new Thread(new Runnable() { + + @Override + public void run() { + StateMonitor.this.run(); + } + }, "StateMonitor"); + thread.setDaemon(true); + thread.start(); + } + + /** Returns a metric consumer for jDisc which will write metrics back to this */ + public MetricConsumer newMetricConsumer() { + StateMetricConsumer consumer = new StateMetricConsumer(); + consumers.add(consumer); + return consumer; + } + + /** Returns the last snapshot taken of the metrics in this system */ + public MetricSnapshot snapshot() { + return snapshot; + } + + /** Returns the interval between each metrics snapshot used by this */ + public long getSnapshotIntervalMillis() { return snapshotIntervalMs; } + + boolean checkTime() { + long now = timer.currentTimeMillis(); + if (now < lastSnapshotTimeMs + snapshotIntervalMs) { + return false; + } + snapshot = createSnapshot(lastSnapshotTimeMs, now); + lastSnapshotTimeMs = now; + return true; + } + + private void run() { + log.finest("StateMonitor started."); + try { + while (!Thread.interrupted()) { + checkTime(); + Thread.sleep((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis()); + } + } catch (InterruptedException e) { + + } + log.finest("StateMonitor stopped."); + } + + private MetricSnapshot createSnapshot(long fromMillis, long toMillis) { + MetricSnapshot snapshot = new MetricSnapshot(fromMillis, toMillis, TimeUnit.MILLISECONDS); + for (StateMetricConsumer consumer : consumers) { + snapshot.add(consumer.createSnapshot()); + } + updateNames(snapshot); + return snapshot; + } + + private void updateNames(MetricSnapshot current) { + TreeSet<String> seen = new TreeSet<>(); + for (Map.Entry<MetricDimensions, MetricSet> dimensionAndMetric : current) { + for (Map.Entry<String, MetricValue> nameAndMetric : dimensionAndMetric + .getValue()) { + seen.add(nameAndMetric.getKey()); + } + } + synchronized (valueNames) { + for (String name : valueNames) { + if (!seen.contains(name)) { + current.add((MetricDimensions) StateMetricConsumer.NULL_CONTEXT, name, 0); + } + } + valueNames.addAll(seen); + } + } + + @Override + public void deconstruct() { + thread.interrupt(); + try { + thread.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (thread.isAlive()) { + log.warning("StateMonitor failed to terminate within 5 seconds of interrupt signal. Ignoring."); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java new file mode 100644 index 00000000000..b92dc098004 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.jdisc.state; + +import com.yahoo.osgi.annotation.ExportPackage; + +/** + * Metrics implementation for jDisc. This consumes metrics over the jDisc metric API + * and makes these available for in-process consumption through + * {@link com.yahoo.container.jdisc.state.StateMonitor#snapshot}, + * and off-process through a jDisc handler. + */
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/messagebus/handler/.gitignore b/container-core/src/main/java/com/yahoo/container/messagebus/handler/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/messagebus/handler/.gitignore diff --git a/container-core/src/main/java/com/yahoo/container/messagebus/testutil/.gitignore b/container-core/src/main/java/com/yahoo/container/messagebus/testutil/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/messagebus/testutil/.gitignore diff --git a/container-core/src/main/java/com/yahoo/container/osgi/AbstractRpcAdaptor.java b/container-core/src/main/java/com/yahoo/container/osgi/AbstractRpcAdaptor.java new file mode 100644 index 00000000000..77d76cb20ea --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/osgi/AbstractRpcAdaptor.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.osgi; + +import com.yahoo.jrt.Supervisor; + +/** + * Helper class for optional RPC adaptors in the Container. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public abstract class AbstractRpcAdaptor { + + public abstract void bindCommands(Supervisor supervisor); + +} diff --git a/container-core/src/main/java/com/yahoo/container/osgi/ContainerRpcAdaptor.java b/container-core/src/main/java/com/yahoo/container/osgi/ContainerRpcAdaptor.java new file mode 100644 index 00000000000..64a75df5770 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/osgi/ContainerRpcAdaptor.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.osgi; + +import com.yahoo.jrt.Acceptor; +import com.yahoo.jrt.ErrorCode; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.jrt.slobrok.api.Register; +import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.net.LinuxInetAddress; +import com.yahoo.log.LogLevel; +import com.yahoo.osgi.Osgi; +import com.yahoo.yolean.Exceptions; +import org.osgi.framework.Bundle; + +import java.net.UnknownHostException; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * An rpc adaptor to container Osgi commands. + * + * @author bratseth + */ +public class ContainerRpcAdaptor extends AbstractRpcAdaptor { + + private static final Logger log = Logger.getLogger(ContainerRpcAdaptor.class.getName()); + + private Acceptor acceptor; + private final Supervisor supervisor; + + private final Osgi osgi; + private final String hostname; + + private Optional<String> slobrokId = Optional.empty(); + private Optional<Register> slobrokRegistrator = Optional.empty(); + + public ContainerRpcAdaptor(Osgi osgi) { + this.osgi = osgi; + supervisor = new Supervisor(new Transport()); + + try { + this.hostname = LinuxInetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + throw new RuntimeException("Failed gettting local hostname", e); + } + + bindCommands(supervisor); + } + + public void list(Request request) { + try { + StringBuilder buffer=new StringBuilder("Installed bundles:"); + for (Bundle bundle : osgi.getBundles()) { + if (bundle.getSymbolicName().equals("system.bundle")) continue; + buffer.append("\n"); + buffer.append(bundle.getSymbolicName()); + buffer.append(" ("); + buffer.append(bundle.getLocation()); + buffer.append(")"); + } + request.returnValues().add(new StringValue(buffer.toString())); + } + catch (Exception e) { + request.setError(ErrorCode.METHOD_FAILED,Exceptions.toMessageString(e)); + } + } + + public void bindCommands(Supervisor supervisor) { + supervisor.addMethod(new Method("list","","s",this,"list")); + } + + public synchronized void listen(int port) { + Spec spec = new Spec(port); + try { + acceptor = supervisor.listen(spec); + log.log(LogLevel.DEBUG, "Added new rpc server listening at" + " port '" + port + "'."); + } catch (ListenFailedException e) { + throw new RuntimeException("Could not create rpc server listening on " + spec, e); + } + } + + public synchronized void setSlobrokId(String slobrokId) { + this.slobrokId = Optional.of(slobrokId); + } + + public synchronized void registerInSlobrok(List<String> slobrokConnectionSpecs) { + shutdownSlobrokRegistrator(); + + if (slobrokConnectionSpecs.isEmpty()) { + return; + } + + if (!slobrokId.isPresent()) { + throw new AssertionError("Slobrok id must be set first"); + } + + SlobrokList slobrokList = new SlobrokList(); + slobrokList.setup(slobrokConnectionSpecs.stream().toArray(String[]::new)); + + Spec mySpec = new Spec(hostname, acceptor.port()); + + Register register = new Register(supervisor, slobrokList, mySpec); + register.registerName(slobrokId.get()); + slobrokRegistrator = Optional.of(register); + + log.log(LogLevel.INFO, "Registered name '" + slobrokId.get() + "' at " + mySpec + " with: " + slobrokList); + } + + private synchronized void shutdownSlobrokRegistrator() { + slobrokRegistrator.ifPresent(Register::shutdown); + slobrokRegistrator = Optional.empty(); + } + + public synchronized void shutdown() { + shutdownSlobrokRegistrator(); + + if (acceptor != null) { + acceptor.shutdown().join(); + } + supervisor.transport().shutdown().join(); + } + + + public synchronized void bindRpcAdaptor(AbstractRpcAdaptor adaptor) { + adaptor.bindCommands(supervisor); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/osgi/package-info.java b/container-core/src/main/java/com/yahoo/container/osgi/package-info.java new file mode 100644 index 00000000000..6a271a6ec0f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/osgi/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.osgi; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/package-info.java b/container-core/src/main/java/com/yahoo/container/package-info.java new file mode 100644 index 00000000000..54f3a272d27 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/protect/Error.java b/container-core/src/main/java/com/yahoo/container/protect/Error.java new file mode 100644 index 00000000000..08ef8a0393d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/protect/Error.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.protect; + +/** + * Error codes to use in ErrorMessage instances for container applications. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public enum Error { + + NO_BACKENDS_IN_SERVICE(0), + NULL_QUERY(1), + REQUEST_TOO_LARGE(2), + ILLEGAL_QUERY(3), + INVALID_QUERY_PARAMETER(4), + UNSPECIFIED(5), + ERROR_IN_PLUGIN(6), + INVALID_QUERY_TRANSFORMATION(7), + RESULT_HAS_ERRORS(8), + SERVER_IS_MISCONFIGURED(9), + BACKEND_COMMUNICATION_ERROR(10), + NO_ANSWER_WHEN_PINGING_NODE(11), + TIMEOUT(12), + EMPTY_DOCUMENTS(13), + UNAUTHORIZED(14), + FORBIDDEN(15), + NOT_FOUND(16), + BAD_REQUEST(17), + INTERNAL_SERVER_ERROR(18); + + public final int code; + + Error(int code) { + this.code = code; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/protect/FreezeDetector.java b/container-core/src/main/java/com/yahoo/container/protect/FreezeDetector.java new file mode 100644 index 00000000000..97b8304babc --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/protect/FreezeDetector.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.protect; + +import java.util.Timer; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.container.core.DiagnosticsConfig; + +/** + * Runs and initializes a {@link Watchdog} instance. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @deprecated this is not in use and will be removed in the next major release + */ +@Deprecated +public class FreezeDetector extends AbstractComponent { + + private final Timer timeoutWatchdog; + private final Watchdog watchdog; + + public FreezeDetector(DiagnosticsConfig diagnosticsConfig) { + if (diagnosticsConfig.disabled()) { + timeoutWatchdog = null; + watchdog = null; + } else { + timeoutWatchdog = new Timer("TimeoutWatchdog", true); + watchdog = new Watchdog(diagnosticsConfig.timeoutfraction(), + diagnosticsConfig.minimumqps(), + diagnosticsConfig.shutdown()); + timeoutWatchdog.schedule(watchdog, 10L * 1000L, 100L); + } + } + + public void register(ThreadLocalDirectory<TimeoutRate, Boolean> timeouts) { + if (watchdog == null) { + return; + } + watchdog.addTimeouts(timeouts); + } + + public boolean isBreakdown() { + if (watchdog == null) { + return false; + } + return watchdog.isBreakdown(); + } + + public void unRegister(ThreadLocalDirectory<TimeoutRate, Boolean> timeouts) { + if (watchdog == null) { + return; + } + watchdog.removeTimeouts(timeouts); + } + + @Override + public void deconstruct() { + super.deconstruct(); + if (timeoutWatchdog != null) { + timeoutWatchdog.cancel(); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java new file mode 100644 index 00000000000..7ff077f9f30 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.protect; + +import com.yahoo.protect.Process; + +/** + * An injectable terminator of the Java vm. + * Components that encounters conditions where the vm should be terminator should + * request an instance of this injected. That makes termination testable + * as tests can create subclasses of this which register the termination request + * rather than terminating. + * + * @author bratseth + */ +public class ProcessTerminator { + + /** Logs and dies without taking a thread dump */ + public void logAndDie(String message) { + logAndDie(message, false); + } + + /** + * Logs and dies + * + * @param dumpThreads if true the stack trace of all threads is dumped to the + * log with level info before shutting down + */ + public void logAndDie(String message, boolean dumpThreads) { + Process.logAndDie(message, dumpThreads); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/protect/TimeoutCollector.java b/container-core/src/main/java/com/yahoo/container/protect/TimeoutCollector.java new file mode 100644 index 00000000000..ee2f6419423 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/protect/TimeoutCollector.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.protect; + +import com.yahoo.concurrent.ThreadLocalDirectory.Updater; + +/** + * Allocator and glue for sampling timeouts in SearchHandler. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @deprecated this is not in use and will be removed in the next major release + */ +@Deprecated +public final class TimeoutCollector implements Updater<TimeoutRate, Boolean> { + + @Override + public TimeoutRate createGenerationInstance(TimeoutRate previous) { + return new TimeoutRate(); + } + + @Override + public TimeoutRate update(TimeoutRate current, Boolean x) { + current.addQuery(x); + return current; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/protect/TimeoutRate.java b/container-core/src/main/java/com/yahoo/container/protect/TimeoutRate.java new file mode 100644 index 00000000000..79e52b49183 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/protect/TimeoutRate.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.protect; + +/** + * Helper class to account for measuring how many queries times outs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @deprecated this is not in use and will be removed in the next major release + */ +@Deprecated +public final class TimeoutRate { + + private int timeouts = 0; + private int total = 0; + + public void addQuery(Boolean timeout) { + if (timeout) { + timeouts += 1; + } + total += 1; + } + + public void merge(TimeoutRate other) { + timeouts += other.timeouts; + total += other.total; + } + + public double timeoutFraction() { + if (total == 0) { + return 0.0d; + } else { + return ((double) timeouts) / ((double) total); + } + } + + public int getTotal() { + return total; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/protect/Watchdog.java b/container-core/src/main/java/com/yahoo/container/protect/Watchdog.java new file mode 100644 index 00000000000..b86da523a0a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/protect/Watchdog.java @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.protect; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TimerTask; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.log.LogLevel; +import com.yahoo.protect.Process; + +/** + * Watchdog for a frozen process, too many timeouts, etc. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @deprecated this is not in use and will be removed in the next major release + */ +@Deprecated +class Watchdog extends TimerTask { + + public static final String FREEZEDETECTOR_DISABLE = "vespa.freezedetector.disable"; + Logger log = Logger.getLogger(Watchdog.class.getName()); + private long lastRun = 0L; + private long lastQpsCheck = 0L; + // Local copy to avoid ever _reading_ the volatile version + private boolean breakdownCopy = false; + private volatile boolean breakdown; + // The fraction of queries which must time out to view the QRS as being + // in breakdown + private final double timeoutThreshold; + // The minimal QPS to care about timeoutThreshold + private final int minimalQps; + private final boolean disableSevereBreakdownCheck; + private final List<ThreadLocalDirectory<TimeoutRate, Boolean>> timeoutRegistry = new ArrayList<>(); + private final boolean shutdownIfFrozen; + + Watchdog(double timeoutThreshold, int minimalQps, boolean shutdownIfFrozen) { + this.timeoutThreshold = timeoutThreshold; + this.minimalQps = minimalQps; + if (System.getProperty(FREEZEDETECTOR_DISABLE) != null) { + disableSevereBreakdownCheck = true; + } else { + disableSevereBreakdownCheck = false; + } + this.shutdownIfFrozen = shutdownIfFrozen; + } + + @Override + public void run() { + long now = System.currentTimeMillis(); + if (lastRun != 0L) { + severeBreakdown(now); + queryTimeouts(now); + } else { + lastQpsCheck = now; + } + lastRun = now; + } + + private void severeBreakdown(final long now) { + if (disableSevereBreakdownCheck) { + return; + } + if (now - lastRun < 5000L) { + return; + } + + threadStackMessage(); + + if (shutdownIfFrozen) { + Process.logAndDie("Watchdog timer meant to run ten times per second" + + " not run for five seconds or more." + + " Assuming severe failure or overloaded node, shutting down container."); + } else { + log.log(LogLevel.ERROR, + "A watchdog meant to run 10 times a second has not been invoked for 5 seconds." + + " This usually means this machine is swapping or otherwise severely overloaded."); + } + } + + private void threadStackMessage() { + log.log(LogLevel.INFO, "System seems unresponsive, performing full thread dump for diagnostics."); + threadDump(); + log.log(LogLevel.INFO, "End of diagnostic thread dump."); + } + + private void threadDump() { + try { + Map<Thread, StackTraceElement[]> allStackTraces = Thread.getAllStackTraces(); + for (Map.Entry<Thread, StackTraceElement[]> e : allStackTraces.entrySet()) { + Thread t = e.getKey(); + StackTraceElement[] stack = e.getValue(); + StringBuilder forOneThread = new StringBuilder(); + int initLen; + forOneThread.append("Stack for thread: ").append(t.getName()).append(": "); + initLen = forOneThread.length(); + for (StackTraceElement s : stack) { + if (forOneThread.length() > initLen) { + forOneThread.append(" "); + } + forOneThread.append(s.toString()); + } + log.log(LogLevel.INFO, forOneThread.toString()); + } + } catch (Exception e) { + // just give up... + } + } + + private void queryTimeouts(final long now) { + // only check query timeout every 10s + if (now - lastQpsCheck < 10000L) { + return; + } else { + lastQpsCheck = now; + } + + final TimeoutRate globalState = new TimeoutRate(); + synchronized (timeoutRegistry) { + for (ThreadLocalDirectory<TimeoutRate, Boolean> timeouts : timeoutRegistry) { + final List<TimeoutRate> threadStates = timeouts.fetch(); + for (final TimeoutRate t : threadStates) { + globalState.merge(t); + } + } + } + if (globalState.timeoutFraction() > timeoutThreshold && globalState.getTotal() > (10 * minimalQps)) { + setBreakdown(true); + log.log(Level.WARNING, "Too many queries timed out. Assuming container is in breakdown."); + } else { + if (!breakdown()) { + return; + } + setBreakdown(false); + log.log(Level.WARNING, "Fewer queries timed out. Assuming container is no longer in breakdown."); + } + } + + private void setBreakdown(final boolean state) { + breakdown = state; + breakdownCopy = state; + } + + private boolean breakdown() { + return breakdownCopy; + } + + boolean isBreakdown() { + return breakdown; + } + + void addTimeouts(ThreadLocalDirectory<TimeoutRate, Boolean> t) { + synchronized (timeoutRegistry) { + timeoutRegistry.add(t); + } + } + + void removeTimeouts(ThreadLocalDirectory<TimeoutRate, Boolean> timeouts) { + synchronized (timeoutRegistry) { + timeoutRegistry.remove(timeouts); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/protect/package-info.java b/container-core/src/main/java/com/yahoo/container/protect/package-info.java new file mode 100644 index 00000000000..a026833cc21 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/protect/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.container.protect; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/servlet/ServletProvider.java b/container-core/src/main/java/com/yahoo/container/servlet/ServletProvider.java new file mode 100644 index 00000000000..252c401fea5 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/servlet/ServletProvider.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.servlet; + +import javax.servlet.Servlet; + +import com.yahoo.container.di.componentgraph.Provider; +import org.eclipse.jetty.servlet.ServletHolder; + +/** + * @author stiankri + */ +public class ServletProvider implements Provider<ServletHolder> { + + private ServletHolder servletHolder; + + public ServletProvider(Servlet servlet, ServletConfigConfig servletConfigConfig) { + servletHolder = new ServletHolder(servlet); + + servletConfigConfig.map().forEach( (key, value) -> + servletHolder.setInitParameter(key, value) + ); + } + + @Override + public ServletHolder get() { + return servletHolder; + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/servlet/package-info.java b/container-core/src/main/java/com/yahoo/container/servlet/package-info.java new file mode 100644 index 00000000000..caf1bdd3910 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/servlet/package-info.java @@ -0,0 +1,6 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.servlet; + +import com.yahoo.osgi.annotation.ExportPackage; + diff --git a/container-core/src/main/java/com/yahoo/container/xml/bind/JAXBContextFactory.java b/container-core/src/main/java/com/yahoo/container/xml/bind/JAXBContextFactory.java new file mode 100644 index 00000000000..b28128bd4f0 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/bind/JAXBContextFactory.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.bind; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +/** + * Container components can take an instance of this class as a constructor argument, + * to get a new instance injected by the container framework. There is usually no + * need to create an instance with this class' constructor. + * <p> + * This factory is needed because the JAXBContext needs a user defined context path, + * which means that it cannot be created at the time the container creates its + * component graph. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author gjoranv + * @since 5.3 + */ +public class JAXBContextFactory { + public static final String FACTORY_CLASS = "com.sun.xml.internal.bind.v2.ContextFactory"; + + /** + * Returns a new JAXBContext for the context path defined by the given list of classes. + * @return A new JAXBContext. + * @param classes One class per package that contains schema derived classes and/or + * java to schema (JAXB-annotated) mapped classes + */ + public JAXBContext newInstance(Class<?>... classes) { + return newInstance(getContextPath(classes), classes[0].getClassLoader()); + } + + // TODO: guard against adding the same package more than once + static String getContextPath(Class<?>... classes) { + if (classes == null || classes.length == 0) { + throw new IllegalArgumentException("Empty package list."); + } + StringBuilder contextPath = new StringBuilder(); + for (Class<?> clazz : classes) { + contextPath + .append(clazz.getPackage().getName()) + .append(':'); + } + contextPath.deleteCharAt(contextPath.length() - 1); + return contextPath.toString(); + } + + private static JAXBContext newInstance(String contextPath, ClassLoader classLoader) { + System.setProperty(JAXBContext.JAXB_CONTEXT_FACTORY, FACTORY_CLASS); + try { + return JAXBContext.newInstance(contextPath, classLoader); + } catch (JAXBException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/bind/package-info.java b/container-core/src/main/java/com/yahoo/container/xml/bind/package-info.java new file mode 100644 index 00000000000..1783bceb610 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/bind/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.container.xml.bind; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java new file mode 100644 index 00000000000..096c77bb744 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class DatatypeFactoryProvider implements Provider<DatatypeFactory> { + public static final String FACTORY_CLASS = DatatypeFactory.DATATYPEFACTORY_IMPLEMENTATION_CLASS; + + @Override + public DatatypeFactory get() { + try { + return DatatypeFactory.newInstance( + FACTORY_CLASS, + this.getClass().getClassLoader()); + } catch (DatatypeConfigurationException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java new file mode 100644 index 00000000000..ad0840c067b --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.parsers.DocumentBuilderFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class DocumentBuilderFactoryProvider implements Provider<DocumentBuilderFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"; + + @Override + public DocumentBuilderFactory get() { + return DocumentBuilderFactory.newInstance(FACTORY_CLASS, + this.getClass().getClassLoader()); + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/JAXBContextFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/JAXBContextFactoryProvider.java new file mode 100644 index 00000000000..3c988c4d397 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/JAXBContextFactoryProvider.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.container.xml.bind.JAXBContextFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class JAXBContextFactoryProvider implements Provider<JAXBContextFactory> { + public static final String FACTORY_CLASS = JAXBContextFactory.class.getName(); + + @Override + public JAXBContextFactory get() { + return new JAXBContextFactory(); + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java new file mode 100644 index 00000000000..5cf921884c5 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.parsers.SAXParserFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class SAXParserFactoryProvider implements Provider<SAXParserFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"; + + @Override + public SAXParserFactory get() { + return SAXParserFactory.newInstance(FACTORY_CLASS, + this.getClass().getClassLoader()); + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java new file mode 100644 index 00000000000..df9ca229ba4 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.XMLConstants; +import javax.xml.validation.SchemaFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class SchemaFactoryProvider implements Provider<SchemaFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.validation.XMLSchemaFactory"; + + @Override + public SchemaFactory get() { + return SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI, + FACTORY_CLASS, + this.getClass().getClassLoader()); + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java new file mode 100644 index 00000000000..7bddadc6c7d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.transform.TransformerFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class TransformerFactoryProvider implements Provider<TransformerFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xalan.internal.xsltc.trax.TransformerFactoryImpl"; + + @Override + public TransformerFactory get() { + return TransformerFactory.newInstance(FACTORY_CLASS, + this.getClass().getClassLoader()); + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java new file mode 100644 index 00000000000..cd9e18f385a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.stream.XMLEventFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class XMLEventFactoryProvider implements Provider<XMLEventFactory> { + public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.events.XMLEventFactoryImpl"; + + @Override + public XMLEventFactory get() { + System.setProperty("javax.xml.stream.XMLEventFactory", FACTORY_CLASS); + // NOTE: In case the newFactory(String, ClassLoader) is used, XMLEventFactory treats the string as classname. + return XMLEventFactory.newFactory(); + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java new file mode 100644 index 00000000000..2bc9941cb13 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.stream.XMLInputFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class XMLInputFactoryProvider implements Provider<XMLInputFactory> { + private static final String INPUT_FACTORY_INTERFACE = XMLInputFactory.class.getName(); + public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.XMLInputFactoryImpl"; + + @Override + public XMLInputFactory get() { + //ugly, but must be done + System.setProperty(INPUT_FACTORY_INTERFACE, FACTORY_CLASS); + + // NOTE: In case the newFactory(String, ClassLoader) is used, + // the given class loader is ignored if the system property is set! + return XMLInputFactory.newFactory(); + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java new file mode 100644 index 00000000000..f0a349a1cf3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.stream.XMLOutputFactory; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class XMLOutputFactoryProvider implements Provider<XMLOutputFactory> { + public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.XMLOutputFactoryImpl"; + @Override + public XMLOutputFactory get() { + System.setProperty("javax.xml.stream.XMLOutputFactory", FACTORY_CLASS); + + // NOTE: In case the newFactory(String, ClassLoader) is used, XMLOutputFactory treats the string as system + // property name. Also, the given class loader is ignored if the property is set! + return XMLOutputFactory.newFactory(); + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java new file mode 100644 index 00000000000..62eba5d4de9 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.xml.providers; + +import com.yahoo.container.di.componentgraph.Provider; + +import javax.xml.xpath.XPathFactory; +import javax.xml.xpath.XPathFactoryConfigurationException; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.29 + */ +public class XPathFactoryProvider implements Provider<XPathFactory> { + public static final String FACTORY_CLASS = "com.sun.org.apache.xpath.internal.jaxp.XPathFactoryImpl"; + + @Override + public XPathFactory get() { + try { + return XPathFactory.newInstance(XPathFactory.DEFAULT_OBJECT_MODEL_URI, + FACTORY_CLASS, + this.getClass().getClassLoader()); + } catch (XPathFactoryConfigurationException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void deconstruct() { } +} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/package-info.java b/container-core/src/main/java/com/yahoo/container/xml/providers/package-info.java new file mode 100644 index 00000000000..5b112d54e2f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.container.xml.providers; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/language/provider/SimpleLinguisticsProvider.java b/container-core/src/main/java/com/yahoo/language/provider/SimpleLinguisticsProvider.java new file mode 100644 index 00000000000..8be317e719a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/language/provider/SimpleLinguisticsProvider.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.provider; + +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.language.Linguistics; + +/** + * Provides simple linguistics if no linguistics component has been explicitly configured + * (dependency injection will fallback to providers if no components of the requested type is found). + * + * @author bratseth + */ +public class SimpleLinguisticsProvider implements Provider<Linguistics> { + + private final Linguistics linguistics; + + public SimpleLinguisticsProvider() { + linguistics = new SimpleLinguistics(); + } + + @Override + public Linguistics get() { return linguistics; } + + @Override + public void deconstruct() {} + +} diff --git a/container-core/src/main/java/com/yahoo/language/provider/package-info.java b/container-core/src/main/java/com/yahoo/language/provider/package-info.java new file mode 100644 index 00000000000..ed1d75515b5 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/language/provider/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.language.provider; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/metrics/package-info.java b/container-core/src/main/java/com/yahoo/metrics/package-info.java new file mode 100644 index 00000000000..50374144683 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/metrics/package-info.java @@ -0,0 +1,6 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** Exported config package */ +@ExportPackage +package com.yahoo.metrics; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java b/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java new file mode 100644 index 00000000000..806dbaf0c42 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.test.NonWorkingOsgiFramework; +import org.osgi.framework.Bundle; +import org.osgi.framework.ServiceReference; + +import java.util.Collections; +import java.util.List; + +/** + * @author tonytv + */ +public class MockOsgi extends NonWorkingOsgiFramework implements Osgi { + + @Override + public Bundle[] getBundles() { + return new Bundle[0]; + } + + @Override + public Bundle getBundle(ComponentSpecification bundleId) { + return null; + } + + @Override + public List<Bundle> install(String absolutePath) { + return Collections.emptyList(); + } + + @Override + public void uninstall(Bundle bundle) { + } + + @Override + public void refreshPackages() { + } + +} diff --git a/container-core/src/main/java/com/yahoo/osgi/Osgi.java b/container-core/src/main/java/com/yahoo/osgi/Osgi.java new file mode 100644 index 00000000000..4090b8512b3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/osgi/Osgi.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import org.osgi.framework.Bundle; +import org.osgi.framework.ServiceReference; + +import java.util.List; + +/** + * @author tonytv + */ +public interface Osgi { + + Bundle[] getBundles(); + + Bundle getBundle(ComponentSpecification bundleId); + + List<Bundle> install(String absolutePath); + + void uninstall(Bundle bundle); + + void refreshPackages(); + +} diff --git a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java new file mode 100644 index 00000000000..986dde43f6a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.application.OsgiFramework; +import org.osgi.framework.Bundle; +import org.osgi.framework.BundleException; +import org.osgi.framework.ServiceReference; + +import java.util.List; +import java.util.logging.Logger; + +/** + * @author tonytv + */ +public class OsgiImpl implements Osgi { + + private static final Logger log = Logger.getLogger(OsgiImpl.class.getName()); + + private final OsgiFramework jdiscOsgi; + + public OsgiImpl(OsgiFramework jdiscOsgi) { + this.jdiscOsgi = jdiscOsgi; + } + + @Override + public Bundle[] getBundles() { + List<Bundle> bundles = jdiscOsgi.bundles(); + return bundles.toArray(new Bundle[bundles.size()]); + } + + + public Class<Object> resolveClass(BundleInstantiationSpecification spec) { + Bundle bundle = getBundle(spec.bundle); + if (bundle != null) { + return resolveFromBundle(spec, bundle); + } else { + return resolveFromClassPath(spec); + } + } + + @SuppressWarnings("unchecked") + private static Class<Object> resolveFromClassPath(BundleInstantiationSpecification spec) { + try { + return (Class<Object>) Class.forName(spec.classId.getName()); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Could not create a component with id '" + spec.classId.getName() + + "'. Tried to load class directly, since no bundle was found for spec: " + spec.bundle + + ". If a bundle with the same name is installed, there is a either a version mismatch" + + " or the installed bundle's version contains a qualifier string."); + } + } + + @SuppressWarnings("unchecked") + private static Class<Object> resolveFromBundle(BundleInstantiationSpecification spec, Bundle bundle) { + try { + ensureBundleActive(bundle); + return (Class<Object>) bundle.loadClass(spec.classId.getName()); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Could not load class '" + spec.classId.getName() + + "' from bundle " + bundle, e); + } + } + + private static void ensureBundleActive(Bundle bundle) throws IllegalStateException { + final int state = bundle.getState(); + Throwable cause = null; + if (state != Bundle.ACTIVE) { + try { + //Get the reason why the bundle isn't active. + //Do not change this method to not fail if start is successful without carefully analyzing + //why there are non-active bundles. + bundle.start(); + } catch (BundleException e) { + cause = e; + } + throw new IllegalStateException("Bundle " + bundle + " is not active. State=" + state + ".", cause); + } + } + + /** + * Returns the bundle of a given name having the highest matching version + * + * @param id the id of the component to return. May not include a version, or include + * an underspecified version, in which case the highest (mathcing) version which + * does not contain a qualifier is returned + * @return the bundle match having the highest version, or null if there was no matches + */ + public Bundle getBundle(ComponentSpecification id) { + Bundle highestMatch=null; + for (Bundle bundle : getBundles()) { + assert bundle.getSymbolicName() != null : "ensureHasBundleSymbolicName not called during installation"; + + if ( ! bundle.getSymbolicName().equals(id.getName())) continue; + if ( ! id.getVersionSpecification().matches(versionOf(bundle))) continue; + + if (highestMatch==null || versionOf(highestMatch).compareTo(versionOf(bundle))<0) + highestMatch=bundle; + } + return highestMatch; + } + + /** returns the version of a bundle, as specified by Bundle-Version in the manifest */ + private static com.yahoo.component.Version versionOf(Bundle bundle) { + Object bundleVersion=bundle.getHeaders().get("Bundle-Version"); + if (bundleVersion==null) return com.yahoo.component.Version.emptyVersion; + return new com.yahoo.component.Version(bundleVersion.toString()); + } + + @Override + public List<Bundle> install(String absolutePath) { + try { + return jdiscOsgi.installBundle(normalizeLocation(absolutePath)); + } catch (BundleException e) { + throw new RuntimeException(e); + } + } + + private static String normalizeLocation(String location) { + if (location.indexOf(':')<0) + location="file:" + location; + return location; + } + + @Override + public void uninstall(Bundle bundle) { + try { + bundle.uninstall(); + } catch (BundleException e) { + throw new RuntimeException(e); + } + } + + @Override + public void refreshPackages() { + jdiscOsgi.refreshPackages(); + } +} diff --git a/container-core/src/main/java/com/yahoo/osgi/package-info.java b/container-core/src/main/java/com/yahoo/osgi/package-info.java new file mode 100644 index 00000000000..007b799b244 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/osgi/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.osgi; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/osgi/provider/.gitignore b/container-core/src/main/java/com/yahoo/osgi/provider/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-core/src/main/java/com/yahoo/osgi/provider/.gitignore diff --git a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java new file mode 100644 index 00000000000..a463cfa2ba1 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java @@ -0,0 +1,263 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.handler; + +import com.google.inject.Inject; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.chain.ChainedComponent; +import com.yahoo.component.chain.model.ChainsModelBuilder; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.container.jdisc.ContentChannelOutputStream; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.VespaHeaders; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.processing.Processor; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.execution.Execution; +import com.yahoo.processing.execution.ResponseReceiver; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.processing.rendering.AsynchronousSectionedRenderer; +import com.yahoo.processing.rendering.ProcessingRenderer; +import com.yahoo.processing.rendering.Renderer; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.processing.request.ErrorMessage; +import com.yahoo.processing.request.Properties; +import com.yahoo.processing.response.Data; +import com.yahoo.processing.response.DataList; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; + +import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry; + +/** + * Superclass of handlers invoking some kind of processing chain. + * <p> + * COMPONENT: The type of the processing components of which this executes a chain + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @author tonyv + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1.6 + */ +public abstract class AbstractProcessingHandler<COMPONENT extends Processor> extends LoggingRequestHandler { + + private final static CompoundName freezeListenerKey =new CompoundName("processing.freezeListener"); + + public final static String DEFAULT_RENDERER_ID = "default"; + + private final Executor renderingExecutor; + + private ChainRegistry<COMPONENT> chainRegistry; + + private final ComponentRegistry<Renderer> renderers; + + private final Renderer defaultRenderer; + + public AbstractProcessingHandler(ChainRegistry<COMPONENT> chainRegistry, + ComponentRegistry<Renderer> renderers, + Executor executor, + AccessLog accessLog, + Metric metric) { + super(executor, accessLog, metric, true); + renderingExecutor = executor; + this.chainRegistry = chainRegistry; + this.renderers = renderers; + + // Default is the one with id "default", or the ProcessingRenderer if there is no such renderer + Renderer defaultRenderer = renderers.getComponent(ComponentSpecification.fromString(DEFAULT_RENDERER_ID)); + if (defaultRenderer == null) { + defaultRenderer = new ProcessingRenderer(); + renderers.register(ComponentId.fromString(DEFAULT_RENDERER_ID), defaultRenderer); + } + this.defaultRenderer = defaultRenderer; + } + + public AbstractProcessingHandler(ChainRegistry<COMPONENT> chainRegistry, + ComponentRegistry<Renderer> renderers, + Executor executor, + AccessLog accessLog) { + this(chainRegistry, renderers, executor, accessLog, null); + } + + public AbstractProcessingHandler(ChainsConfig processingChainsConfig, + ComponentRegistry <COMPONENT> chainedComponents, + ComponentRegistry<Renderer> renderers, + Executor executor, + AccessLog accessLog) { + this(processingChainsConfig, chainedComponents, renderers, executor, accessLog, null); + } + + @Inject + public AbstractProcessingHandler(ChainsConfig processingChainsConfig, + ComponentRegistry<COMPONENT> chainedComponents, + ComponentRegistry<Renderer> renderers, + Executor executor, + AccessLog accessLog, + Metric metric) { + this(createChainRegistry(processingChainsConfig, chainedComponents), renderers, executor, accessLog, metric); + } + + /** Throws UnsupportedOperationException: Call handle(request, channel instead) */ + @Override + public HttpResponse handle(HttpRequest request) { + throw new UnsupportedOperationException("Call handle(request, channel) instead"); + } + + @Override + @SuppressWarnings("unchecked") + public HttpResponse handle(HttpRequest request, ContentChannel channel) { + com.yahoo.processing.Request processingRequest = new com.yahoo.processing.Request(); + populate("", request.propertyMap(), processingRequest.properties()); + populate("context", request.getJDiscRequest().context(), processingRequest.properties()); + processingRequest.properties().set(Request.JDISC_REQUEST, request); + + FreezeListener freezeListener = new FreezeListener(processingRequest, renderers, defaultRenderer, channel, renderingExecutor); + processingRequest.properties().set(freezeListenerKey, freezeListener); + + Chain<COMPONENT> chain = chainRegistry.getComponent(resolveChainId(processingRequest.properties())); + if (chain == null) + throw new IllegalArgumentException("Chain '" + processingRequest.properties().get("chain") + "' not found"); + Execution execution = createExecution(chain, processingRequest); + freezeListener.setExecution(execution); + Response processingResponse = execution.process(processingRequest); + + return freezeListener.getHttpResponse(processingResponse); + } + + public Execution createExecution(Chain<COMPONENT> chain, Request processingRequest) { + int traceLevel = processingRequest.properties().getInteger("tracelevel", 0); + return Execution.createRoot(chain, traceLevel, new Execution.Environment<>(chainRegistry)); + } + + public ChainRegistry<COMPONENT> getChainRegistry() { return chainRegistry; } + + public ComponentRegistry<Renderer> getRenderers() { return renderers; } + + /** + * For internal use only + */ + @SuppressWarnings("unchecked") + public Renderer<Response> getRendererCopy(ComponentSpecification spec) { + Renderer<Response> renderer = getRenderers().getComponent(spec); + if (renderer == null) throw new IllegalArgumentException("No renderer with spec: " + spec); + return perRenderingCopy(renderer); + } + + private static Renderer<Response> perRenderingCopy(Renderer<Response> renderer) { + Renderer<Response> copy = renderer.clone(); + copy.init(); + return copy; + } + + private static Renderer selectRenderer(com.yahoo.processing.Request processingRequest, + ComponentRegistry<Renderer> renderers, Renderer defaultRenderer) { + Renderer renderer = null; + // TODO: Support setting a particular renderer instead of just selecting + // by name? + String rendererId = processingRequest.properties().getString("format"); + if (rendererId != null && !"".equals(rendererId)) { + renderer = renderers.getComponent(ComponentSpecification.fromString(rendererId)); + if (renderer == null) + processingRequest.errors().add(new ErrorMessage("Could not find renderer","Requested '" + rendererId + + "', has " + renderers.allComponents())); + } + if (renderer == null) + renderer = defaultRenderer; + return renderer; + } + + private static <COMPONENT extends ChainedComponent> ChainRegistry<COMPONENT> createChainRegistry( + ChainsConfig processingChainsConfig, + ComponentRegistry<COMPONENT> availableComponents) { + + ChainRegistry<COMPONENT> chainRegistry = new ChainRegistry<>(); + prepareChainRegistry(chainRegistry, ChainsModelBuilder.buildFromConfig(processingChainsConfig), availableComponents); + chainRegistry.freeze(); + return chainRegistry; + } + + private String resolveChainId(Properties properties) { + return properties.getString(Request.CHAIN,"default"); + } + + private void populate(String prefixName,Map<String,?> parameters,Properties properties) { + CompoundName prefix = new CompoundName(prefixName); + for (Map.Entry<String,?> entry : parameters.entrySet()) + properties.set(prefix.append(entry.getKey()),entry.getValue()); + } + + private static class FreezeListener implements Runnable, ResponseReceiver { + + /** Used to create the renderer */ + private final com.yahoo.processing.Request request; + private final ComponentRegistry<Renderer> renderers; + private final Renderer defaultRenderer; + private final ContentChannel channel; + private final Executor renderingExecutor; + + /** Used to render */ + private Execution execution; + private Response response; + + /** The renderer used in this, or null if not created yet */ + private Renderer<Response> renderer = null; + + public FreezeListener(com.yahoo.processing.Request request, ComponentRegistry<Renderer> renderers, + Renderer defaultRenderer, ContentChannel channel, Executor renderingExecutor) { + this.request = request; + this.renderers = renderers; + this.defaultRenderer = defaultRenderer; + this.channel = channel; + this.renderingExecutor = renderingExecutor; + } + + /** Expected to be called once before run is called */ + @Override + public void setResponse(Response response) { this.response = response; } + + /** Expected to be called once before run is called */ + public void setExecution(Execution execution) { this.execution = execution; } + + /** Returns and lazily creates the renderer of this. May be called even if run is never called. */ + public Renderer getRenderer() { + if (renderer == null) + renderer = perRenderingCopy(selectRenderer(request, renderers, defaultRenderer)); + return renderer; + } + + /** Returns and lazily creates the http response of this. May be called even if run is never called. */ + private HttpResponse getHttpResponse(Response processingResponse) { + int status = 200; // true status is determined asynchronously in ProcessingResponse.complete() + return new ProcessingResponse(status, request, processingResponse, getRenderer(), renderingExecutor, execution); + } + + @Override + public void run() { + if (execution == null || response == null) + throw new NullPointerException("Uninitialized freeze listener"); + + if (channel instanceof LazyContentChannel) + ((LazyContentChannel)channel).setHttpResponse(getHttpResponse(response)); + + // Render if we have a renderer capable of it + if (getRenderer() instanceof AsynchronousSectionedRenderer) { + ((AsynchronousSectionedRenderer) getRenderer()).renderBeforeHandover(new ContentChannelOutputStream(channel), response, execution, request); + } + } + + } + +} diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingHandler.java new file mode 100644 index 00000000000..6e9d9df9557 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingHandler.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.handler; + +import com.google.inject.Inject; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Metric; +import com.yahoo.processing.Processor; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.processing.rendering.Renderer; + +import java.util.concurrent.Executor; + +/** + * A jDisc request handler which invokes a processing chain to produce the response. + * + * @author tonytv + * @since 5.1.7 + */ +public class ProcessingHandler extends AbstractProcessingHandler<Processor> { + public ProcessingHandler(ChainRegistry<Processor> chainRegistry, + ComponentRegistry<Renderer> renderers, + Executor executor, + AccessLog accessLog, + Metric metric) { + super(chainRegistry, renderers, executor, accessLog, metric); + } + + public ProcessingHandler(ChainRegistry<Processor> chainRegistry, + ComponentRegistry<Renderer> renderers, + Executor executor, + AccessLog accessLog) { + super(chainRegistry, renderers, executor, accessLog); + } + + public ProcessingHandler(ChainsConfig processingChainsConfig, + ComponentRegistry<Processor> chainedComponents, + ComponentRegistry<Renderer> renderers, + Executor executor, + AccessLog accessLog) { + super(processingChainsConfig, chainedComponents, renderers, executor, accessLog); + } + + @Inject + public ProcessingHandler(ChainsConfig processingChainsConfig, + ComponentRegistry<Processor> chainedComponents, + ComponentRegistry<Renderer> renderers, + Executor executor, + AccessLog accessLog, + Metric metric) { + super(processingChainsConfig, chainedComponents, renderers, executor, accessLog, metric); + } +} diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingResponse.java b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingResponse.java new file mode 100644 index 00000000000..efca279cd38 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingResponse.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.handler; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Executor; + +import com.google.common.collect.ImmutableList; +import com.yahoo.container.jdisc.AsyncHttpResponse; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.VespaHeaders; +import com.yahoo.container.logging.AccessLogEntry; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.execution.Execution; +import com.yahoo.processing.execution.Execution.Trace.LogValue; +import com.yahoo.processing.rendering.AsynchronousRenderer; +import com.yahoo.processing.rendering.Renderer; +import com.yahoo.processing.request.ErrorMessage; +import com.yahoo.processing.response.Data; +import com.yahoo.processing.response.DataList; + +/** + * A response from running a request through processing. This response is just a + * wrapper of the knowhow needed to render the Response from processing. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1.12 + */ +public class ProcessingResponse extends AsyncHttpResponse { + + private final com.yahoo.processing.Request processingRequest; + private final com.yahoo.processing.Response processingResponse; + private final Executor renderingExecutor; + private final Execution execution; + private final Renderer renderer; + + /** True if the return status has been set explicitly and should not be further changed */ + private boolean explicitStatusSet = false; + + @SuppressWarnings("unchecked") + public ProcessingResponse( + int status, + final com.yahoo.processing.Request processingRequest, + final com.yahoo.processing.Response processingResponse, + final Renderer renderer, + final Executor renderingExecutor, final Execution execution) { + super(status); + this.processingRequest = processingRequest; + this.processingResponse = processingResponse; + this.renderingExecutor = renderingExecutor; + this.execution = execution; + this.renderer = renderer; + } + + @SuppressWarnings("unchecked") + @Override + public void render(final OutputStream stream, final ContentChannel channel, + final CompletionHandler completionHandler) throws IOException { + if (renderer instanceof AsynchronousRenderer) { + AsynchronousRenderer asyncRenderer = (AsynchronousRenderer)renderer; + asyncRenderer.setNetworkWiring(channel, completionHandler); + } + renderer.render(stream, processingResponse, execution, processingRequest); + // the stream is closed in AsynchronousSectionedRenderer, after all data + // has arrived + } + + @Override + public String getContentType() { + return renderer.getMimeType(); + } + + @Override + public String getCharacterEncoding() { + return renderer.getEncoding(); + } + + @Override + public void complete() { + // Add headers + addHeadersAndStatusFrom(processingResponse.data()); + + if ( ! explicitStatusSet) { + // Set status from errors TODO: This could be decomplicated a bit + List<ErrorMessage> errors = flattenErrors(processingResponse); + boolean isSuccess = !(processingResponse.data().asList().isEmpty() && !errors.isEmpty()); // NOT success if ( no data AND are errors ) + setStatus(getHttpResponseStatus(isSuccess, processingRequest, errors.size() == 0 ? null : errors.get(0), errors)); + } + } + + /** + * This sets header and status from special Data items used for the purpose. + * Do both at once to avoid traversing the data tree twice. + */ + @SuppressWarnings("unchecked") + private void addHeadersAndStatusFrom(DataList<Data> dataList) { + for (Data data : dataList.asList()) { + if (data instanceof ResponseHeaders) { + headers().addAll(((ResponseHeaders) data).headers()); + } + else if ( ! explicitStatusSet && (data instanceof ResponseStatus)) { + setStatus(((ResponseStatus)data).code()); + explicitStatusSet = true; + } + else if (data instanceof DataList) { + addHeadersAndStatusFrom((DataList) data); + } + } + } + + private List<ErrorMessage> flattenErrors(Response processingResponse) { + Set<ErrorMessage> errors = flattenErrors(null, processingResponse.data()); + if (errors == null) return Collections.emptyList(); + return ImmutableList.copyOf(errors); + } + + @SuppressWarnings("unchecked") + private Set<ErrorMessage> flattenErrors(Set<ErrorMessage> errors, Data data) { + if (data.request() == null) return Collections.EMPTY_SET; // Not allowed, but handle anyway + errors = addTo(errors, data.request().errors()); + + if (data instanceof DataList) { + for (Data item : ((DataList<Data>) data).asList()) + errors = flattenErrors(errors, item); + } + + return errors; + } + + private Set<ErrorMessage> addTo(Set<ErrorMessage> allErrors, List<ErrorMessage> errors) { + if (errors.isEmpty()) return allErrors; + + if (allErrors == null) + allErrors = new LinkedHashSet<>(); + allErrors.addAll(errors); + return allErrors; + } + + private int getHttpResponseStatus(boolean isSuccess, Request request, + ErrorMessage mainError, List<ErrorMessage> errors) { + if (isBenchmarking(request)) return VespaHeaders.getEagerErrorStatus(mainError,errors.iterator()); + return VespaHeaders.getStatus(isSuccess, mainError, errors.iterator()); + } + + private boolean isBenchmarking(Request request) { + com.yahoo.container.jdisc.HttpRequest httpRequest = (com.yahoo.container.jdisc.HttpRequest)request.properties().get(Request.JDISC_REQUEST); + if (httpRequest == null) return false; + return VespaHeaders.benchmarkOutput(httpRequest); + } + + @Override + public Iterable<LogValue> getLogValues() { + return execution.trace()::logValueIterator; + } + +} diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java new file mode 100644 index 00000000000..b02c0fcccdb --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.handler; + +import com.google.common.annotations.Beta; +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.container.logging.AccessLogInterface; +import com.yahoo.processing.Processor; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.processing.rendering.Renderer; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +/** + * A helper for making processing requests and rendering their responses. + * Create an instance of this to test making processing requests and get the response or response data. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.21 + */ +@Beta +public class ProcessingTestDriver extends RequestHandlerTestDriver { + + private final ProcessingHandler processingHandler; + + public ProcessingTestDriver(Collection<Chain<Processor>> chains) { + this(chains, new ComponentRegistry<Renderer>()); + } + public ProcessingTestDriver(String binding, Collection<Chain<Processor>> chains) { + this(chains, new ComponentRegistry<Renderer>()); + } + @SafeVarargs + public ProcessingTestDriver(Chain<Processor> ... chains) { + this(Arrays.asList(chains), new ComponentRegistry<Renderer>()); + } + @SafeVarargs + public ProcessingTestDriver(String binding, Chain<Processor> ... chains) { + this(binding, Arrays.asList(chains), new ComponentRegistry<Renderer>()); + } + public ProcessingTestDriver(Collection<Chain<Processor>> chains, ComponentRegistry<Renderer> renderers) { + this(createProcessingHandler(chains, renderers, AccessLog.voidAccessLog())); + } + public ProcessingTestDriver(String binding, Collection<Chain<Processor>> chains, ComponentRegistry<Renderer> renderers) { + this(binding, createProcessingHandler(chains, renderers, AccessLog.voidAccessLog())); + } + public ProcessingTestDriver(ProcessingHandler processingHandler) { + super(processingHandler); + this.processingHandler = processingHandler; + } + public ProcessingTestDriver(String binding, ProcessingHandler processingHandler) { + super(binding, processingHandler); + this.processingHandler = processingHandler; + } + + public ProcessingTestDriver(Chain<Processor> chain, AccessLogInterface accessLogInterface) { + this(createProcessingHandler( + Collections.singleton(chain), + new ComponentRegistry<Renderer>(), + createAccessLog(accessLogInterface))); + } + + private static AccessLog createAccessLog(AccessLogInterface accessLogInterface) { + ComponentRegistry<AccessLogInterface> componentRegistry = new ComponentRegistry<>(); + componentRegistry.register(ComponentId.createAnonymousComponentId("access-log"), accessLogInterface); + componentRegistry.freeze(); + + return new AccessLog(componentRegistry); + } + + private static ProcessingHandler createProcessingHandler( + Collection<Chain<Processor>> chains, + ComponentRegistry<Renderer> renderers, + AccessLog accessLog) { + Executor executor = Executors.newSingleThreadExecutor(); + + ChainRegistry<Processor> registry = new ChainRegistry<>(); + for (Chain<Processor> chain : chains) + registry.register(chain.getId(), chain); + return new ProcessingHandler(registry, renderers, executor, accessLog); + } + + /** Returns the processing handler of this */ + public ProcessingHandler processingHandler() { return processingHandler; } + +} diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ResponseHeaders.java b/container-core/src/main/java/com/yahoo/processing/handler/ResponseHeaders.java new file mode 100644 index 00000000000..d86ef31441e --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/handler/ResponseHeaders.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.handler; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.processing.Request; +import com.yahoo.processing.response.AbstractData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Holds a set of headers which will be added to the Processing response. + * A Response may contain multiple such data objects, and all of them will be added to the response. + * + * @author bratseth + * @since 5.1.23 + */ +public class ResponseHeaders extends AbstractData { + + private final Map<String,List<String>> headers; + + /** + * Creates a response headers object with a set of headers. + * + * @param headers the headers to copy into this object + */ + public ResponseHeaders(Map<String,List<String>> headers, Request request) { + super(request); + this.headers = ImmutableMap.copyOf(headers); + } + + /** Returns an unmodifiable map of the response headers of this */ + public Map<String,List<String>> headers() { return headers; } + +} diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ResponseStatus.java b/container-core/src/main/java/com/yahoo/processing/handler/ResponseStatus.java new file mode 100644 index 00000000000..0778cae91c7 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/handler/ResponseStatus.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.handler; + +import com.yahoo.processing.Request; +import com.yahoo.processing.response.AbstractData; + +/** + * <p>A data item holding a response HTTP status code. + * If this is present in a Response it will determine the HTTP status of the response (when returned over HTTP), + * regardless of any errors present in the result which might otherwise determine the response status.</p> + * + * <p>If several ResponseStatus instances are present, the first one encountered by a depth-first search through + * the data composite tree will be used.</p> + * + * <p>Note that this must be added to the response before any response data is writable to take effect.</p> + * + * @author bratseth + */ +public class ResponseStatus extends AbstractData { + + /** A http status code */ + private final int code; + + public ResponseStatus(int code, Request request) { + super(request); + this.code = code; + } + + /** Returns the code of this */ + public int code() { return code; } + + @Override + public String toString() { + return "HTTP response " + code; + } + +} diff --git a/container-core/src/main/java/com/yahoo/processing/handler/package-info.java b/container-core/src/main/java/com/yahoo/processing/handler/package-info.java new file mode 100644 index 00000000000..19f9fd38d9e --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/handler/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.processing.handler; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/processing/processors/RequestPropertyTracer.java b/container-core/src/main/java/com/yahoo/processing/processors/RequestPropertyTracer.java new file mode 100644 index 00000000000..195ecc73e9f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/processors/RequestPropertyTracer.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.processors; + +import com.yahoo.processing.Processor; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.execution.Execution; + +import java.util.Map; + +/** + * A processor which adds the current content of the Request.properties() to + * the trace before calling the next processor, if traceLevel is 4 or more. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.17 + */ +public class RequestPropertyTracer extends Processor { + + @Override + public Response process(Request request, Execution execution) { + if (execution.trace().getTraceLevel()<4) return execution.process(request); + + StringBuilder b = new StringBuilder("{"); + for (Map.Entry<String,Object> property : request.properties().listProperties().entrySet()) + b.append(property.getKey()).append(": '").append(property.getValue()).append("',"); + b.setLength(b.length()-1); // remove last comma + b.append("}"); + execution.trace().trace(b.toString(),4); + return execution.process(request); + } + +} diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousRenderer.java new file mode 100644 index 00000000000..e04902251ee --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousRenderer.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.rendering; + +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.processing.Response; + +import java.util.concurrent.Executor; + +/** + * Superclass of all asynchronous renderers. + * Subclasses this to receive an executor and the network wiring necessary to respectively + * run callback listeners and close up the channel when the response is complete. + * + * @author bratseth + */ +public abstract class AsynchronousRenderer <RESPONSE extends Response> extends Renderer<RESPONSE> { + + /** + * Exposes JDisc wiring to ensure asynchronous cleanup. + * + * @param channel the channel to the client receiving the response + * @param completionHandler the JDisc completion handler which will be invoked at the end + * of the rendering + * @throws IllegalStateException if attempted invoked more than once + */ + public abstract void setNetworkWiring(ContentChannel channel, CompletionHandler completionHandler); + +} diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java new file mode 100644 index 00000000000..6ebb328fafc --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -0,0 +1,572 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.rendering; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.log.LogLevel; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.execution.Execution; +import com.yahoo.processing.response.AbstractDataList; +import com.yahoo.processing.response.Data; +import com.yahoo.processing.response.DataList; +import com.yahoo.processing.response.Ordered; +import com.yahoo.processing.response.Streamed; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Helper class to implement processing API Response renderers. This renderer + * will walk the data tree and call the appropriate render methods as it + * progresses. Nodes with the same parent branch will be rendered in the order + * in which the data is ready for consumption. + * + * <p> + * This API assumes all data should be rendered. Choosing which data should be + * rendered is the responsibility of the processing chains. + * </p> + * + * @author Steinar Knutsen + * @author Einar M R Rosenvinge + * @author bratseth + */ +public abstract class AsynchronousSectionedRenderer<RESPONSE extends Response> extends AsynchronousRenderer<RESPONSE> { + + /** + * Invoked once at the beginning of rendering a response. This assigns the + * stream to be used throughput the rendering. Subsequent calls must use the + * same stream. + * + * @param stream + * the stream to render to in this and all subsequent calls. + * @throws IOException + * passed on from the stream + */ + public abstract void beginResponse(OutputStream stream) throws IOException; + + /** + * Invoked at the beginning of each data list, including the implicit, + * outermost one in the response. + * + * @throws IOException passed on from the stream + * @param list the data list which now will be rendered + */ + public abstract void beginList(DataList<?> list) throws IOException; + + /** + * Invoked for each leaf node in the data tree + * + * @param data the leaf node to render + * @throws IOException passed on from the stream + */ + public abstract void data(Data data) throws IOException; + + /** + * Invoked at the end of each data list, including the implicit, outermost + * one in the response. + * + * @param list the data list which now has no more data items to render + * @throws IOException passed on from the stream + */ + public abstract void endList(DataList<?> list) throws IOException; + + /** + * Invoked once at the end of rendering a response. + * + * @throws IOException passed on from the stream + */ + public abstract void endResponse() throws IOException; + + private static final Logger logger = Logger.getLogger(AsynchronousSectionedRenderer.class.getName()); + + // NOTE: Renderers are *prototype objects* - a new instance is created for each rendering by invoking clone + // calling init() and then render(). + // Hence any field which is not reinitialized in init() or render() will be *reused* in all rendering operations + // across all threads! + + /** The stack of listeners to ancestor datalist completions above the current one */ + private Deque<DataListListener> dataListListenerStack; + + private boolean beforeHandoverMode; + private OutputStream stream; + private RESPONSE response; + private Execution execution; + private boolean clientClosed; + + // This MUST be created in the init() method - see comment above + private Object singleThreaded; + + // Rendering threads should never block. + // Burst traffic may add work faster than we can complete it, so use an unbounded queue. + private static final ThreadPoolExecutor renderingExecutor = createExecutor(); + private static ThreadPoolExecutor createExecutor() { + int threadCount = Runtime.getRuntime().availableProcessors(); + ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, 1L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + ThreadFactoryFactory.getThreadFactory("rendering")); + executor.prestartAllCoreThreads(); + return executor; + } + + private SettableFuture<Boolean> success; + + private ContentChannel channel; + private CompletionHandler completionHandler; + private boolean networkIsInitialized; + + private boolean isInitialized; + + /** + * Create an renderer instance not yet associated with any request + * processing or network for easy subclassing. It is the handler's + * responsibility to wire in the resources needed by a renderer + * before use. + */ + public AsynchronousSectionedRenderer() { + isInitialized = false; + } + + /** + * <p>Render this response using the renderer's own threads and return a future indicating whether the rendering + * was successful. The data list tree will be traversed asynchronously, and + * the pertinent methods will be called as data becomes available.</p> + * + * <p>If rendering fails, the exception causing this will be wrapped in an + * ExecutionException and thrown from blocked calls to Future.get()</p> + * + * @return a future indicating whether rendering was successful + */ + @Override + public final ListenableFuture<Boolean> render(OutputStream stream, RESPONSE response, + Execution execution, Request request) { + if (beforeHandoverMode) { // rendering has already started or is already complete + beforeHandoverMode = false; + if ( ! dataListListenerStack.isEmpty() && + dataListListenerStack.getFirst().list.incoming().isComplete()) { + // We're not waiting for async completion, so kick off more rendering due to the implicit complete + // (return Response from chain) causing this method to be called + getExecutor().execute(dataListListenerStack.getFirst()); + } + return success; + } + else { // This is the start of rendering + return startRender(stream, response, execution, request); + } + } + + @Override + public void deconstruct() { + super.deconstruct(); + renderingExecutor.shutdown(); + try { + if (renderingExecutor.awaitTermination(30, TimeUnit.SECONDS)) + throw new RuntimeException("Rendering thread pool did not shutdown in 30 seconds"); + } + catch (InterruptedException e) { + // return + } + } + + /** + * Initiate rendering before handover to rendering threads. + * This is rendering which happens before the Response is returned from the main chain, + * caused by freezing of DataLists. + * At this point the worker thread still owns the Response, so all this rendering must happen + * on the caller thread invoking freeze (that is, on the thread calling this). + */ + public final ListenableFuture<Boolean> renderBeforeHandover(OutputStream stream, RESPONSE response, + Execution execution, Request request) { + beforeHandoverMode = true; + if (!isInitialized) throw new IllegalStateException("render() invoked before init()."); + + return startRender(stream, response, execution, request); + } + + private ListenableFuture<Boolean> startRender(OutputStream stream, RESPONSE response, + Execution execution, Request request) { + this.response = response; + this.stream = stream; + this.execution = execution; + DataListListener parentOfTopLevelListener = new DataListListener(new ParentOfTopLevel(request,response.data()), null); + dataListListenerStack.addFirst(parentOfTopLevelListener); + success = SettableFuture.create(); + try { + getExecutor().execute(parentOfTopLevelListener); + } catch (RejectedExecutionException e) { + parentOfTopLevelListener.closeIO(e); + } + return success; + } + + /** + * Returns the executor in which to execute a listener. + * Before handover this *must* be the calling thread, because listeners are free to modify the dataList. + * After handover it can be any thread in the renderer pool. + * Note that as some listeners may be set up before handover and executed after, it is possible that some rendering + * inadvertently work ends up in async data producing threads in some cases. + */ + Executor getExecutor() { + return beforeHandoverMode ? MoreExecutors.sameThreadExecutor() : renderingExecutor; + } + + /** The outermost execution which was run to create the response to render. */ + public Execution getExecution() { return execution; } + + /** The response render callbacks are generated from. */ + public Response getResponse() { return response; } + + /** Returns whether the client this is rendering to has closed the connection */ + protected boolean clientClosed() { return clientClosed; } + + /** This hook is called once when the renderer detects that the client has closed the connection */ + protected void onClientClosed() { } + + /** + * How deep into the tree of nested data lists the callback currently is. + * beginList() is invoked after this this is increased, and endList() is + * invoked before it is decreased. + * + * @return an integer of 1 or above + */ + public int getRecursionLevel() { + return dataListListenerStack.size()-1; + } + + /** + * For internal use: Expose JDisc wiring to ensure asynchronous cleanup. + * + * @param channel the channel to the client receiving the response + * @param completionHandler the JDisc completion handler which will be invoked at the end + * of the rendering + * @throws IllegalStateException if attempted invoked more than once + */ + @Override + public final void setNetworkWiring(ContentChannel channel, CompletionHandler completionHandler) { + if (networkIsInitialized) + throw new IllegalStateException("Network wiring already set and can only be set once."); + + this.channel = channel; + this.completionHandler = completionHandler; + networkIsInitialized = true; + } + + /** + * Do per instance initialization. If overriding this in a subclass, not + * invoking it in the subclass' implementation will most likely cause the + * rendering to fail with an exception. + */ + @Override + public void init() { + beforeHandoverMode = false; + clientClosed = false; + singleThreaded = new Object(); + dataListListenerStack = new ArrayDeque<>(); + networkIsInitialized = false; + isInitialized = true; + } + + /** + * A listener to async completion of a data list. + * All rendering is done by callbacks to this, even in the sync case (where the callback happens immediately). + * One such listener is registered for every data list encountered during rendering. + * Only the last one registered is allowed to run at any point in time, as that one renders the lowest level + * list not yet completed (rendering, of course is depth first). + * <p> + * A stack of registered renderers is maintained to maintain this constraint. + * <p> + * A renderer maintains state sufficient to allow it to resume rendering at a later stage. + * This is to be able to render child lists to completion before completing rendering of the parent list. + * In addition, this feature is used by DataListeners (see below). + */ + private class DataListListener extends RendererListener { + + /** The index of the next data item where rendering should be initiated in this list */ + private int currentIndex = 0; + + /** Children of this which has started rendering but not yet completed */ + private int uncompletedChildren = 0; + + private boolean listStartIsRendered = false; + + /** The list which this is listening to */ + private final DataList list; + + /** The listener to the parent of this list, or null if this is the root */ + private final DataListListener parent; + + public DataListListener(DataList list, DataListListener parent) { + this.list = list; + this.parent = parent; + } + + @Override + protected void render() throws IOException, InterruptedException, ExecutionException { + if (dataListListenerStack.peekFirst() != this) + return; // This listens to some ancestor of the current list, do this later + if (beforeHandoverMode && ! list.isFrozen()) + return; // Called on completion of a list which is not frozen yet - hold off until frozen + + if ( ! beforeHandoverMode) + list.complete().get(); // trigger completion if not done already to invoke any listeners on that event + boolean startedRendering = renderData(); + if ( ! startedRendering || uncompletedChildren > 0) return; // children must render to completion first + if (list.complete().isDone()) // might not be when in before handover mode + endListLevel(); + else + stream.flush(); + } + + private void endListLevel() throws IOException { + endRenderLevel(list); + stream.flush(); + dataListListenerStack.removeFirst(); + if (parent != null) + parent.childCompleted(); + } + + /** Called each time a direct child of this completed. */ + private void childCompleted() { + uncompletedChildren--; + + if (uncompletedChildren>0) return; + if (list.incoming().isComplete()) // i) if the parent had completed earlier, render it now, see ii) + run(); + } + + /** + * Resumes rendering data from the current position. + * Called both on completion (by this), and when new data is available (from the new data listener). + * + * @return whether this started rendering + */ + @SuppressWarnings("unchecked") + private boolean renderData() throws IOException { + if (dataListListenerStack.peekFirst() != this) return false; // This listens to some ancestor of the current list, do this later + renderDataListStart(); + + // Add newly arrived data, and as a consequence run data listeners + for (Object data : list.incoming().drain()) + list.add((Data) data); + + renderDataList(list); + return true; + } + + void renderDataListStart() throws IOException { + if ( ! listStartIsRendered) { + if (list instanceof ParentOfTopLevel) + beginResponse(stream); + else + beginList(list); + listStartIsRendered = true; + } + } + + /** + * Renders a list + */ + private void renderDataList(DataList list) throws IOException { + final boolean ordered = isOrdered(list); + while (currentIndex < list.asList().size()) { + Data data = list.get(currentIndex++); + if (data instanceof DataList) { + listenTo((DataList)data, ordered && isStreamed((DataList)data)); + uncompletedChildren++; + if (ordered) + return; // ii) Resumed by the child list when done, see i) + } + else { + data(data); + } + } + } + + private void listenTo(DataList subList, boolean listenToNewDataAdded) throws IOException { + DataListListener listListener = new DataListListener(subList,this); + dataListListenerStack.addFirst(listListener); + + if (listenToNewDataAdded) + subList.incoming().addNewDataListener(new DataListener(listListener), getExecutor()); + + flushIfLikelyToSuspend(subList); + + subList.addFreezeListener(listListener, getExecutor()); + subList.complete().addListener(listListener, getExecutor()); + subList.incoming().completed().addListener(listListener, getExecutor()); + } + + private boolean isOrdered(DataList dataList) { + if (! (dataList instanceof Ordered)) + return true; // all lists are ordered by default + return ((Ordered)dataList).isOrdered(); + } + + private boolean isStreamed(DataList dataList) { + if (! (dataList instanceof Streamed)) + return true; // all lists are streamed by default + return ((Streamed)dataList).isStreamed(); + } + + private void endRenderLevel(DataList<?> current) throws IOException { + if (current instanceof ParentOfTopLevel) { + endResponse(); + closeIO(null); + } + else { + endList(current); + } + } + + private void closeIO(Exception failed) { + IOException closeException = null; + + try { + stream.close(); + } catch (IOException e) { + closeException = e; + logger.log(LogLevel.WARNING, "Exception caught while closing stream to client.", e); + } finally { + if (failed != null) { + success.setException(failed); + } else if (closeException != null) { + success.setException(closeException); + } else { + success.set(true); + } + if (channel != null) { + channel.close(completionHandler); + } + } + } + + @Override + public String toString() { + return "listener to " + list; + } + + } + + /** + * A data listener is invoked every time new data is available in an incoming list, such that this data + * can be rendered before completion of the entire list (streaming). + * <p> + * One data renderer is registered for every incoming data list. + * It will delegate to the data list listener of the same list such that the correct rendering progress state is + * shared between rendering here and from the completion listener. + */ + private class DataListener extends RendererListener { + + /** The listener to completion of the data list this listens to new data in. */ + private DataListListener dataListListener; + + public DataListener(DataListListener dataListListener) { + this.dataListListener = dataListListener; + } + + protected void render() throws IOException, InterruptedException, ExecutionException { + dataListListener.renderData(); + flushIfLikelyToSuspend(dataListListener.list); + } + + } + + private abstract class RendererListener implements Runnable { + + protected abstract void render() throws IOException, InterruptedException, ExecutionException; + + public void run() { + try { + synchronized (singleThreaded) { + try { + render(); + } catch (Exception e) { + Level level = LogLevel.WARNING; + if ((e instanceof IOException)) { + level = LogLevel.DEBUG; + if ( ! clientClosed) { + clientClosed = true; + onClientClosed(); + } + } + if (logger.isLoggable(level)) { + logger.log(level, "Exception caught during response rendering.", e); + } + if (channel != null) { + try { + channel.close(completionHandler); + } catch (Exception ignored) { + } + } + success.setException(e); + } + } + } catch (Error e) { + // We are in free-range thread land, and a hanging container is really no fun at all. + com.yahoo.protect.Process.logAndDie("Caught fatal error during rendering.", e); + } + } + + protected void flushIfLikelyToSuspend(DataList list) throws IOException { + // If the listener is not complete, we will (likely) suspend rendering + if ( ! list.incoming().isComplete()) stream.flush(); + } + + } + + /** + * This must be pushed on the stack first to get things started off, given that the stack is expected to + * contain the parent of each element (including the topmost) + */ + private static class ParentOfTopLevel extends AbstractDataList { + + private DataList trueTopLevel; + + public ParentOfTopLevel(Request request,DataList trueTopLevel) { + super(request); + this.trueTopLevel = trueTopLevel; + freeze(); + } + + @Override + public Data add(Data data) { + throw new IllegalStateException("We're not supposed to add to this"); + } + + @Override + public void addDataListener(Runnable listener) { + throw new IllegalStateException("We're not supposed to listen to or add to this"); + } + + @Override + public Data get(int index) { + if (index>0) throw new IndexOutOfBoundsException(); + return trueTopLevel; + } + + @Override + public List<Data> asList() { + return Collections.<Data>singletonList(trueTopLevel); + } + + @Override + public String toString() { + return "ParentOfTopLevel"; + } + + } + +} diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/ProcessingRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/ProcessingRenderer.java new file mode 100644 index 00000000000..ba2d34eabd7 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/rendering/ProcessingRenderer.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.rendering; + +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.handler.ResponseHeaders; +import com.yahoo.processing.handler.ResponseStatus; +import com.yahoo.processing.request.ErrorMessage; +import com.yahoo.processing.response.Data; +import com.yahoo.processing.response.DataList; +import com.yahoo.text.JSONWriter; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.IdentityHashMap; +import java.util.Map; + +/** + * The default renderer for processing responses. Renders a response in JSON. + * This can be overridden to specify particular rendering of leaf Data elements. + * This default implementation renders the toString of each element. + * + * @author bratseth + */ +public class ProcessingRenderer extends AsynchronousSectionedRenderer<Response> { + + private Map<Request,Request> renderedRequests = new IdentityHashMap<>(); + + private JSONWriter jsonWriter; + + /** The current nesting level */ + private int level; + + @Override + public void init() { + super.init(); + level = 0; + } + + @Override + public final void beginResponse(OutputStream stream) throws IOException { + jsonWriter = new JSONWriter(stream); + } + + @Override + public final void endResponse() throws IOException { + } + + @Override + public final void beginList(DataList<?> list) throws IOException { + if (level>0) + jsonWriter.beginArrayValue(); + + jsonWriter.beginObject(); + + if (level==0) + renderTrace(); + + if ( ! list.request().errors().isEmpty() && ! rendered(list.request())) { + jsonWriter.beginField("errors"); + jsonWriter.beginArray(); + for (ErrorMessage error : list.request().errors()) { + if (renderedRequests == null) + renderedRequests = new IdentityHashMap<>(); + renderedRequests.put(list.request(),list.request()); + jsonWriter.beginArrayValue(); + if (error.getCause() != null) { // render object + jsonWriter.beginObject(); + jsonWriter.beginField("error").value(error.toString()).endField(); + jsonWriter.beginField("stacktrace").value(stackTraceAsString(error.getCause())).endField(); + jsonWriter.endObject(); + } + else { // render string + jsonWriter.value(error.toString()); + } + jsonWriter.endArrayValue(); + } + jsonWriter.endArray(); + jsonWriter.endField(); + } + + jsonWriter.beginField("datalist"); + jsonWriter.beginArray(); + level++; + } + + private String stackTraceAsString(Throwable e) { + StringWriter writer = new StringWriter(); + e.printStackTrace(new PrintWriter(writer)); + return writer.toString(); + } + + private boolean rendered(Request request) { + return renderedRequests != null && renderedRequests.containsKey(request); + } + + @Override + public final void endList(DataList<?> list) throws IOException { + jsonWriter.endArray(); + jsonWriter.endField(); + jsonWriter.endObject(); + if (level>0) + jsonWriter.endArrayValue(); + level--; + } + + @Override + public final void data(Data data) throws IOException { + if (! shouldRender(data)) return; + jsonWriter.beginArrayValue(); + jsonWriter.beginObject(); + jsonWriter.beginField("data"); + renderValue(data,jsonWriter); + jsonWriter.endField(); + jsonWriter.endObject(); + jsonWriter.endArrayValue(); + } + + /** + * Renders the value of a data element. + * This default implementation does writer.fieldValue(data.toString()) + * Override this to render data in application specific ways. + */ + protected void renderValue(Data data,JSONWriter writer) throws IOException { + writer.value(data.toString()); + } + + /** + * Returns whether this data element should be rendered. + * This can be overridden to add new kinds of data which should not be rendered. + * This default implementation returns true unless the data is instanceof ResponseHeaders. + * + * @return true to render it, false to skip completely + */ + protected boolean shouldRender(Data data) { + if (data instanceof ResponseHeaders) return false; + if (data instanceof ResponseStatus) return false; + return true; + } + + @Override + public final String getEncoding() { + return null; + } + + @Override + public final String getMimeType() { + return "application/json"; + } + + private boolean renderTrace() throws IOException { + if (getExecution().trace().getTraceLevel() == 0) return false; + + jsonWriter.beginField("trace"); + try { + getExecution().trace().traceNode().accept(new TraceRenderingVisitor(jsonWriter)); + } catch (WrappedIOException e) { + throw e.getCause(); + } + jsonWriter.endField(); + return true; + } + + private static class TraceRenderingVisitor extends TraceVisitor { + + private JSONWriter jsonWriter; + + public TraceRenderingVisitor(JSONWriter jsonWriter) { + this.jsonWriter = jsonWriter; + } + + @Override + public void entering(TraceNode node) { + try { + jsonWriter.beginArray(); + } + catch (IOException e) { + throw new WrappedIOException(e); + } + } + + @Override + public void leaving(TraceNode node) { + try { + jsonWriter.endArray(); + } + catch (IOException e) { + throw new WrappedIOException(e); + } + } + + @Override + public void visit(TraceNode node) { + if ( ! (node.payload() instanceof String)) return; // skip other info than trace messages + try { + jsonWriter.beginArrayValue(); + if (node.timestamp() != 0) { // render object + jsonWriter.beginObject(); + jsonWriter.beginField("timestamp").value(node.timestamp()).endField(); + jsonWriter.beginField("message").value(node.payload().toString()).endField(); + jsonWriter.endObject(); + } + else { // render string + jsonWriter.value(node.payload().toString()); + } + jsonWriter.endArrayValue(); + } + catch (IOException e) { + throw new WrappedIOException(e); + } + } + + } + + private static class WrappedIOException extends RuntimeException { + private WrappedIOException(IOException cause) { + super(cause); + } + + @Override + public IOException getCause() { + return (IOException) super.getCause(); + } + } +} diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java new file mode 100644 index 00000000000..d20492f998f --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing.rendering; + +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.AbstractComponent; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.execution.Execution; + +import java.io.OutputStream; + +/** + * Renders a response to a stream. The renderers are cloned just before + * rendering, and must therefore obey the following contract: + * + * <ol> + * <li>At construction time, only final members shall be initialized, and these + * must refer to immutable data only.</li> + * <li>State mutated during rendering shall be initialized in the init method.</li> + * </ol> + * + * @author tonytv + * @author Steinar Knutsen + */ +public abstract class Renderer<RESPONSE extends Response> extends AbstractComponent implements Cloneable { + + /** + * Used to create a separate instance for each result to render. + */ + @SuppressWarnings("unchecked") + @Override + public Renderer<RESPONSE> clone() { + return (Renderer<RESPONSE>) super.clone(); + } + + /** + * Initializes the mutable state, see the contract in the class + * documentation. Called on the clone just before rendering. + */ + public void init() { + } + + /** + * Render a response to a stream. The stream also exposes a ByteBuffer API + * for efficient transactions to JDisc. The returned future will throw the + * exception causing failure wrapped in an ExecutionException if rendering + * was not successful. + * + * @param stream + * a stream API bridge to JDisc + * @param response + * the response to render + * @param execution + * the execution which created this response + * @param request + * the request matching the response + * @return a ListenableFuture containing a boolean where true indicates a + * successful rendering + */ + public abstract ListenableFuture<Boolean> render(OutputStream stream, RESPONSE response, + Execution execution, Request request); + + /** + * Name of the output encoding, if applicable. + * + *<p>TODO: ensure null is OK + * + * @return The encoding of the output if applicable, e.g. "utf-8" + */ + public abstract String getEncoding(); + + /** + * The MIME type of the rendered content sent to the client. + * + * @return The mime type of the data written to the writer, e.g. + * "text/plain" + */ + public abstract String getMimeType(); + +} diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/package-info.java b/container-core/src/main/java/com/yahoo/processing/rendering/package-info.java new file mode 100644 index 00000000000..1149b614e7a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/processing/rendering/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.processing.rendering; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/org/json/package-info.java b/container-core/src/main/java/org/json/package-info.java new file mode 100644 index 00000000000..c723ac90210 --- /dev/null +++ b/container-core/src/main/java/org/json/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package org.json; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/resources/config/handlers.cfg b/container-core/src/main/resources/config/handlers.cfg new file mode 100644 index 00000000000..733d822d4ef --- /dev/null +++ b/container-core/src/main/resources/config/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.container.handler.TestHandler diff --git a/container-core/src/main/resources/config/qr-fileserver.cfg b/container-core/src/main/resources/config/qr-fileserver.cfg new file mode 100644 index 00000000000..e43c9d7f004 --- /dev/null +++ b/container-core/src/main/resources/config/qr-fileserver.cfg @@ -0,0 +1 @@ +rootdir "" diff --git a/container-core/src/main/resources/config/qr-logging.cfg b/container-core/src/main/resources/config/qr-logging.cfg new file mode 100644 index 00000000000..0847de467fe --- /dev/null +++ b/container-core/src/main/resources/config/qr-logging.cfg @@ -0,0 +1,44 @@ +logger com.yahoo +speciallog[6] +speciallog[0].name QueryAccessLog +speciallog[0].type file +speciallog[0].filehandler.name QueryAccessLog +speciallog[0].filehandler.pattern ./QueryAccessLog.%Y%m%d%H%M%S +speciallog[0].filehandler.rotation 0 60 ... +speciallog[0].cachehandler.name QueryAccessLog +speciallog[0].cachehandler.size 1000 +speciallog[1].name QueryResultLog +speciallog[1].type cache +speciallog[1].filehandler.name QueryResultLog +speciallog[1].filehandler.pattern ./QueryResultLog.%Y%m%d%H%M%S +speciallog[1].filehandler.rotation 0 60 ... +speciallog[1].cachehandler.name QueryResultLog +speciallog[1].cachehandler.size 1000 +speciallog[2].name ResultImpressionLog +speciallog[2].type file +speciallog[2].filehandler.name ResultImpressionLog +speciallog[2].filehandler.pattern ./ResultImpressionLog.%Y%m%d%H%M%S +speciallog[2].filehandler.rotation 0 60 ... +speciallog[2].cachehandler.name ResultImpressionLog +speciallog[2].cachehandler.size 1000 +speciallog[3].name ServiceEventLog +speciallog[3].type cache +speciallog[3].filehandler.name ServiceEventLog +speciallog[3].filehandler.pattern ./ServiceEventLog.%Y%m%d%H%M%S +speciallog[3].filehandler.rotation 0 60 ... +speciallog[3].cachehandler.name ServiceEventLog +speciallog[3].cachehandler.size 1000 +speciallog[4].name ServiceStatusLog +speciallog[4].type off +speciallog[4].filehandler.name ServiceStatusLog +speciallog[4].filehandler.pattern ./ServiceStatusLog.%Y%m%d%H%M%S +speciallog[4].filehandler.rotation 0 60 ... +speciallog[4].cachehandler.name ServiceStatusLog +speciallog[4].cachehandler.size 1000 +speciallog[5].name ServiceTraceLog +speciallog[5].type parent +speciallog[5].filehandler.name ServiceTraceLog +speciallog[5].filehandler.pattern ./ServiceTraceLog.%Y%m%d%H%M%S +speciallog[5].filehandler.rotation 0 60 ... +speciallog[5].cachehandler.name ServiceTraceLog +speciallog[5].cachehandler.size 1000 diff --git a/container-core/src/main/resources/config/qr.cfg b/container-core/src/main/resources/config/qr.cfg new file mode 100644 index 00000000000..33ac59c0d7e --- /dev/null +++ b/container-core/src/main/resources/config/qr.cfg @@ -0,0 +1,4 @@ +port.search 18081 +port.stats 18085 +maxthreads 200 +requestbuffersize 65536 diff --git a/container-core/src/main/resources/config/statistics.cfg b/container-core/src/main/resources/config/statistics.cfg new file mode 100644 index 00000000000..eb54ab63111 --- /dev/null +++ b/container-core/src/main/resources/config/statistics.cfg @@ -0,0 +1,5 @@ +## Interval between internal sample points measured in minutes +collectioninterval 60 + +## Interval between internal sample points measured in minutes +logginginterval 60 diff --git a/container-core/src/main/resources/configdefinitions/application-metadata.def b/container-core/src/main/resources/configdefinitions/application-metadata.def new file mode 100644 index 00000000000..8ede96f59cf --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/application-metadata.def @@ -0,0 +1,23 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Contains meta info about one deployed application + +version=4 +namespace=container.core + +# The name of the directory that contained the application package +name string default="" + +# The user name that deployed the application +user string default="" + +# The directory the application was deployed from +path string default="" + +# The application timestamp in ms +timestamp long default=0 + +# The md5 hash of the application package contents +checksum string default="" + +# The application generation number +generation long default=0 diff --git a/container-core/src/main/resources/configdefinitions/container-document.def b/container-core/src/main/resources/configdefinitions/container-document.def new file mode 100644 index 00000000000..1bb7ed81245 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/container-document.def @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Container settings for document type management +# +version=1 +namespace=container.core.document + +# A document type name to use a concrete document type for +doctype[].type string + +# The component id of the AbstractConcreteDocumentFactory to use to instantiate the concrete document type +doctype[].factorycomponent string diff --git a/container-core/src/main/resources/configdefinitions/container-http.def b/container-core/src/main/resources/configdefinitions/container-http.def new file mode 100644 index 00000000000..b3a1a8d7d41 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/container-http.def @@ -0,0 +1,18 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=8 +namespace=container.core + +#with this set to false, absolutely NO HTTP services are started +enabled bool default=false + +## Buffer size for incoming requests +requestbuffersize int default=65536 + +## Which port to listen on for search queries +port.search int default=8081 + +## Which interface to bind to. +port.host string default="" + +## Whether to serve files on the same port as search queries. +fileserver.throughsearch bool default=true diff --git a/container-core/src/main/resources/configdefinitions/diagnostics.def b/container-core/src/main/resources/configdefinitions/diagnostics.def new file mode 100644 index 00000000000..cd352751225 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/diagnostics.def @@ -0,0 +1,17 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=2 +namespace=container.core + +## The fraction of queries to time out over a period of 10s to consider +## the system in breakdown. +timeoutfraction double default=0.1 + +## Minimum QPS to consider the system in breakdown. +minimumqps int default=1 + +## Whether to shut down process if in a deadlock situation +shutdown bool default=false + +## Whether to totally disable the detector. Alternative to system property +## which works better in tests. +disabled bool default=false diff --git a/container-core/src/main/resources/configdefinitions/health-monitor.def b/container-core/src/main/resources/configdefinitions/health-monitor.def new file mode 100644 index 00000000000..305ae43ad2d --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/health-monitor.def @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 + +namespace=container.jdisc.config + +# How far between snapshots. 5 minutes by default +snapshot_interval double default=300 diff --git a/container-core/src/main/resources/configdefinitions/http-filter.def b/container-core/src/main/resources/configdefinitions/http-filter.def new file mode 100644 index 00000000000..613be733937 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/http-filter.def @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=container.core.http + +param[].name string +param[].value string + +filterName string default="" +filterClass string default="" diff --git a/container-core/src/main/resources/configdefinitions/metrics-presentation.def b/container-core/src/main/resources/configdefinitions/metrics-presentation.def new file mode 100644 index 00000000000..6efe0a28767 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/metrics-presentation.def @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 + +namespace=metrics + +## Sliding window means present the last n minutes of data, as opposed to +## presenting the newest completed n minute interval. +slidingwindow bool default=true diff --git a/container-core/src/main/resources/configdefinitions/mockservice.def b/container-core/src/main/resources/configdefinitions/mockservice.def new file mode 100644 index 00000000000..6b2c5b771e7 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/mockservice.def @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=container.handler.test + +file file +fileAcquirerTimeout long default=60 diff --git a/container-core/src/main/resources/configdefinitions/qr-logging.def b/container-core/src/main/resources/configdefinitions/qr-logging.def new file mode 100644 index 00000000000..6bff554ede9 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/qr-logging.def @@ -0,0 +1,39 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=4 +namespace=container.core +logger string default="com.yahoo" +# Either QueryAccessLog for a regular Vespa access log, or YApacheAccessLog for a log on yApache format +speciallog[].name string + +# Leave as "" +speciallog[].type string + +speciallog[].filehandler.name string default="" + +# File name patterns supporting the expected time variables +speciallog[].filehandler.pattern string default=".%Y%m%d%H%M%S" + +speciallog[].filehandler.rotation string default="0 60 ..." + +# Defines how file rotation is done. There are two options: +# +# "date" : +# The active log file is given the name resulting from pattern (but in this case "pattern" must yield a +# time-dependent name. In addition, a symlink is created pointing to the newest file. +# The symlink is given the name of the symlink parameter (or the name of this service +# if no parameter is given. +# +# "sequence" : +# The active log file is given the name +# defined by "pattern" (which in this case will likely just be a constant string). +# At rotation, this file is given the name pattern.N where N is 1 + the largest integer found by +# extracting the integers from all files ending by .Integer in the same directory +# +speciallog[].filehandler.rotatescheme string default="date" + +# Use this as the name of the symlink created pointing to the newest file in the "date" naming scheme. +# This is ignored if the sequence naming scheme is used. +speciallog[].filehandler.symlink string default="" + +speciallog[].cachehandler.name string default="" +speciallog[].cachehandler.size int default=1000 diff --git a/container-core/src/main/resources/configdefinitions/qr-searchers.def b/container-core/src/main/resources/configdefinitions/qr-searchers.def new file mode 100644 index 00000000000..e6c6cdc8aef --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/qr-searchers.def @@ -0,0 +1,90 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=49 +namespace=container + +# this file needs more comments + +tag.bold.open string default="<hi>" +tag.bold.close string default="</hi>" +tag.separator string default="<sep />" + +# some searcher specific configuration parameters: + +com.yahoo.prelude.searcher.FieldCollapsingSearcher.collapsesize int default=1 +com.yahoo.prelude.searcher.FieldCollapsingSearcher.extrafactor double default=2.0 +com.yahoo.prelude.searcher.FieldCollapsingSearcher.collapsefield string default="mid" + +com.yahoo.prelude.searcher.BlendingSearcher.numthreads int default=200 +com.yahoo.prelude.searcher.BlendingSearcher.docid string default="" + +com.yahoo.prelude.searcher.JuniperSearcher.source string default="" +com.yahoo.prelude.searcher.JuniperSearcher.defaultdoctype string default="" + +## Query cache that can be placed anywhere in the search chain. Query/Result +## pairs (ie. entries) bigger than maxentrysizebytes will not be cached. +com.yahoo.prelude.searcher.CachingSearcher.cachesizemegabytes int default=100 +com.yahoo.prelude.searcher.CachingSearcher.timetoliveseconds int default=3600 +com.yahoo.prelude.searcher.CachingSearcher.maxentrysizebytes int default=10000 + +com.yahoo.prelude.searcher.XMLStringSearcher.source string default="" + +## Default docsum class the QR server should ask the backend to +## use for representing hints as default. +com.yahoo.prelude.fastsearch.FastSearcher.docsum.defaultclass string default="" + +com.yahoo.prelude.querytransform.PhrasingSearcher.automatonfile string default="" +com.yahoo.prelude.querytransform.NonPhrasingSearcher.automatonfile string default="" +com.yahoo.prelude.querytransform.TermReplacingSearcher.termlist[] string +com.yahoo.prelude.querytransform.CompleteBoostSearcher.source string default="" + +com.yahoo.prelude.querytransform.ExactStringSearcher.source string default="" +com.yahoo.prelude.querytransform.LiteralBoostSearcher.source string default="" +com.yahoo.prelude.querytransform.TermBoostSearcher.source string default="" +com.yahoo.prelude.querytransform.NormalizingSearcher.source string default="" +com.yahoo.prelude.querytransform.StemmingSearcher.source string default="" + +com.yahoo.prelude.statistics.StatisticsSearcher.latencybucketsize int default=30 + + +# here users may add their custom searchers +# (all strings should be class names) +customizedsearchers.rawquery[] string +customizedsearchers.transformedquery[] string +customizedsearchers.blendedresult[] string +customizedsearchers.unblendedresult[] string +customizedsearchers.backend[] string +customizedsearchers.argument[].key string +customizedsearchers.argument[].value string + +## This is for adding searchers which should be below BlendingSearcher, +## but not be linked to any Vespa cluster (directly). +external[].name string +external[].searcher[] string + +# Search cluster specific information. +## Name of search cluster. +searchcluster[].name string + +## Names of search definitions served by search cluster. +searchcluster[].searchdef[] string + +## configid that may be used to get rank-profiles config for the cluster. +searchcluster[].rankprofiles.configid reference default="" + +## Indexing mode of search cluster. +searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME + +## Storage cluster route to use for search cluster if indexingmode is streaming. +searchcluster[].storagecluster.routespec string default="" + +# The available dispatchers on each search cluster +searchcluster[].dispatcher[].host string +searchcluster[].dispatcher[].port int + +## The number of least significant bits of the part id used to specify the +## row number (the rest of the bits specifies the column). Don't touch +## this unless you know why you are doing it. +searchcluster[].rowbits int default=0 + +# Per dispatcher config-id might be nice to have, remove it until needed. +# searchcluster[].dispatcher[].configid reference diff --git a/container-core/src/main/resources/configdefinitions/qr-templates.def b/container-core/src/main/resources/configdefinitions/qr-templates.def new file mode 100644 index 00000000000..a1d5a4f36e0 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/qr-templates.def @@ -0,0 +1,78 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=8 +namespace=container.core + +## Prefix to use in queries to choose a given template +templateset[].urlprefix string + +## The id of the Java class which the given templateset +## should be an instance of. This is only used for implementing +## templates in the Java API instead of Velocity. +templateset[].classid string default="" + +## The symbolic name of the Osgi bundle this template is located in. +## Assumed to be the same as the classid if not set, and is only used +## when classid is used. +templateset[].bundle string default="" + +## The MIME type of a given template +templateset[].mimetype string default="text/html" + +## The character set of a given template +templateset[].encoding string default="iso-8859-1" + +## Not used +templateset[].rankprofile int default=0 + + +## Not used in 1.0 +templateset[].keepalive bool default=false + +## Header template. Always rendered. +templateset[].headertemplate string default="" + +## Footer template. Always rendered. +templateset[].footertemplate string default="" + +## Nohits template. Rendered if there are no hits in the result. +templateset[].nohitstemplate string default="" + +## Hit template. Rendered if there are hits in the result. +templateset[].hittemplate string default="" + +## Error template. Rendered if there is an error condition. This is +## not mutually exclusive with the (no)hit templates as such. +templateset[].errortemplate string default="" + +## Aggregated groups header template. +## Default rendering is used if missing +templateset[].groupsheadertemplate string default="[DEFAULT]" + +## Aggregated range group template. +## Default rendering is used if missing +templateset[].rangegrouptemplate string default="[DEFAULT]" + +## Aggregated exact group template +## Default rendering is used if missing +templateset[].exactgrouptemplate string default="[DEFAULT]" + +## Aggregated groups footer template. +## Default rendering is used if missing +templateset[].groupsfootertemplate string default="[DEFAULT]" + +## Tags used to highlight results, starting a bolded section. +## An empty string means the template should no override what +## was inserted by the search chain. +templateset[].highlightstarttag string default="" +## Tags used to highlight results, ending a bolded section +## An empty string means the template should no override what +## was inserted by the search chain. +templateset[].highlightendtag string default="" +## Tags used to highlight results, separating dynamic snippets +## An empty string means the template should no override what +## was inserted by the search chain. +templateset[].highlightseptag string default="" + +## The summary class to use for this template if there is none +## defined in the query. +templateset[].defaultsummaryclass string default="" diff --git a/container-core/src/main/resources/configdefinitions/qr.def b/container-core/src/main/resources/configdefinitions/qr.def new file mode 100644 index 00000000000..f6715b3e74f --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/qr.def @@ -0,0 +1,33 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=37 +namespace=container + +### All params must be flagged as 'restart' because this config is manually +### retrieved by ConfiguredApplication.start to init the rpc server in +### com.yahoo.container.Server. + +## filedistributor rpc configuration +filedistributor.configid reference default="" restart + +## Is RPC server enabled? +rpc.enabled bool default=false restart + +## RPC server listen port +rpc.port int default=8086 restart + +## Which interface to bind to. +rpc.host string default="" restart + +## The id this service should register itself with in slobrok +rpc.slobrokId string default="" restart + +## Whether to obtain coverage reports by default. This can be overridden with +## the 'reportcoverage' HTTP parameter. +coveragereports bool default=false restart + +## A unique identifier string for this QRS. The only guarantee given is +## this string will be unique for every QRS in a Vespa application. +discriminator string default="qrserver.0" restart + +## Force restart of container on deploy? +restartOnDeploy bool default=false restart diff --git a/container-core/src/main/resources/configdefinitions/servlet-config.def b/container-core/src/main/resources/configdefinitions/servlet-config.def new file mode 100644 index 00000000000..f7b1eccdcd3 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/servlet-config.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=container.servlet +map{} string + diff --git a/container-core/src/main/resources/configdefinitions/threadpool.def b/container-core/src/main/resources/configdefinitions/threadpool.def new file mode 100644 index 00000000000..00a22864a16 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/threadpool.def @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 + +namespace=container.handler + +maxthreads int default=500 + +# The max time the container tolerates having no threads available before it shuts down to +# get out of a bad state. This should be set a bit higher than the expected max execution +# time of each request when in a state of overload, i.e about "worst case execution time*2" +maxThreadExecutionTimeSeconds int default=190 diff --git a/container-core/src/main/resources/configdefinitions/vip-status.def b/container-core/src/main/resources/configdefinitions/vip-status.def new file mode 100644 index 00000000000..f5f1c793724 --- /dev/null +++ b/container-core/src/main/resources/configdefinitions/vip-status.def @@ -0,0 +1,15 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +namespace=container.core + +## If there is a Vespa search backend connected to this container, and that +## backend is out of service, automatically remove this container from VIP +## rotation, ignoring any status file. +noSearchBackendsImpliesOutOfService bool default=true + +## Whether to return hard coded reply or serve "status.html" from disk +accessdisk bool default=false + +## The file to serve as the status file. +## If the paht is relative vespa home is prepended +statusfile string default="share/qrsdocs/status.html" diff --git a/container-core/src/main/scala/com/yahoo/container/handler/observability/Graphviz.scala b/container-core/src/main/scala/com/yahoo/container/handler/observability/Graphviz.scala new file mode 100644 index 00000000000..6403b08f61c --- /dev/null +++ b/container-core/src/main/scala/com/yahoo/container/handler/observability/Graphviz.scala @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.observability + +import com.yahoo.text.Utf8 +import com.yahoo.io.IOUtils +import java.io.{IOException, InputStreamReader, InputStream} +import com.google.common.io.ByteStreams + +/** + * @author tonytv + */ + +object Graphviz { + + @throws(classOf[IOException]) + def runDot(outputType: String, graph: String) = { + val process = Runtime.getRuntime.exec(Array("/bin/sh", "-c", "unflatten -l7 | dot -T" + outputType)) + process.getOutputStream.write(Utf8.toBytes(graph)) + process.getOutputStream.close() + + val result = ByteStreams.toByteArray(process.getInputStream) + process.waitFor() match { + case 0 => result + case 127 => throw new RuntimeException("Couldn't find dot, please ensure that Graphviz is installed.") + case _ => throw new RuntimeException("Failed running dot: " + readString(process.getErrorStream)) + } + } + + private def readString(inputStream: InputStream): String = { + IOUtils.readAll(new InputStreamReader(inputStream, "UTF-8")) + } + +} diff --git a/container-core/src/main/scala/com/yahoo/container/handler/observability/HtmlUtil.scala b/container-core/src/main/scala/com/yahoo/container/handler/observability/HtmlUtil.scala new file mode 100644 index 00000000000..77fd46b0c5b --- /dev/null +++ b/container-core/src/main/scala/com/yahoo/container/handler/observability/HtmlUtil.scala @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.observability + +import xml.{PrettyPrinter, Elem} + + +/** + * @author gjoranv + * @author tonytv + */ +object HtmlUtil { + def link(target: String, anchor: String): Elem = + <a href={target}>{anchor}</a> + + def link(targetAndAnchor: String): Elem = link(targetAndAnchor, targetAndAnchor) + + def unorderedList(items: Elem*) = + <ul> + {items} + </ul> + + def li[T](children: T*) = + <li>{children}</li> + + def h1(name: String) = + <h1>{name}</h1> + + def html(title: String, body: Elem*) = + <html> + <head> + <title>{title}</title> + </head> + <body> + {body} + </body> + </html> + + def prettyPrintXhtml(elem: Elem): String = { + """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">""" + + "\n" + new PrettyPrinter(120, 2).format(elem) + } +} diff --git a/container-core/src/main/scala/com/yahoo/container/handler/observability/OverviewHandler.scala b/container-core/src/main/scala/com/yahoo/container/handler/observability/OverviewHandler.scala new file mode 100644 index 00000000000..7973ee8c811 --- /dev/null +++ b/container-core/src/main/scala/com/yahoo/container/handler/observability/OverviewHandler.scala @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.observability + +import java.util.concurrent.Executor + +import HtmlUtil._ +import OverviewHandler._ +import com.yahoo.container.jdisc.{HttpResponse, HttpRequest, ThreadedHttpRequestHandler} +import com.yahoo.text.Utf8 +import java.io.{PrintWriter, OutputStream} + + +/** + * @author gjoranv + * @author tonytv + */ +class OverviewHandler(executor: Executor) extends ThreadedHttpRequestHandler(executor) { + + @volatile + private var dotGraph: String = _ + + def handle(request: HttpRequest): HttpResponse = { + val path = request.getUri.getPath + + try { + if (path.endsWith("/ComponentGraph")) + handleComponentGraph(request) + else if (path.endsWith("/Overview")) + handleOverview(request) + else + null + } catch { + case e: Exception => errorResponse(e.getMessage) + } + + } + + def handleOverview(request: HttpRequest): HttpResponse = { + new HttpResponse(com.yahoo.jdisc.Response.Status.OK) { + def render(stream: OutputStream) { + stream.write(Utf8.toBytes(overviewPageText)) + } + + override def getContentType: String = { + "text/html" + } + } + } + + def errorResponse(message: String): HttpResponse = { + new HttpResponse(com.yahoo.jdisc.Response.Status.BAD_REQUEST) { + def render(stream: OutputStream) { + new PrintWriter(stream).println(message) + } + } + } + + def handleComponentGraph(request: HttpRequest): HttpResponse = { + var graphType = request.getProperty("type"); + if (graphType == null) + graphType = "text" + + graphType match { + case "text" => textualComponentGraph(dotGraph) + case t if componentGraphTypes.contains(t) => graphicalComponentGraph(t, Graphviz.runDot(graphType,dotGraph)) + case t => errorResponse(t) + } + } + + def textualComponentGraph(dotGraph: String) = + new HttpResponse(com.yahoo.jdisc.Response.Status.OK) { + def render(stream: OutputStream) { + stream.write(Utf8.toBytes(dotGraph)) + } + + override def getContentType: String = { + "text/plain" + } + } + + def graphicalComponentGraph(graphType: String, image: Array[Byte] ): HttpResponse = + new HttpResponse(com.yahoo.jdisc.Response.Status.OK) { + def render(output: OutputStream) { + output.write(image) + } + + override def getContentType: String = { + componentGraphTypes(graphType) + } + } + + def setDotGraph(dotGraph: String) { + this.dotGraph = dotGraph + } +} + +object OverviewHandler { + val componentGraphTypes = Map( + "svg" -> "image/svg+xml", + "png" -> "image/png", + "text" -> "text/plain") + + val overviewPageText = prettyPrintXhtml(overviewPage) + + private def overviewPage = { + def componentGraphLink(graphType: String) = link("Overview/ComponentGraph?type=" + graphType, graphType) + + + html( + title = "Container Overview", + body = + h1("Container Overview"), + unorderedList( + li(link("ApplicationStatus")), + li("ComponentGraph" +: (componentGraphTypes.keys map {componentGraphLink}).toSeq :_*))) + } + +} diff --git a/container-core/src/main/scala/com/yahoo/container/http/filter/FilterChainRepository.scala b/container-core/src/main/scala/com/yahoo/container/http/filter/FilterChainRepository.scala new file mode 100644 index 00000000000..83ca1707390 --- /dev/null +++ b/container-core/src/main/scala/com/yahoo/container/http/filter/FilterChainRepository.scala @@ -0,0 +1,154 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.http.filter + +import com.yahoo.container.core.ChainsConfig +import com.yahoo.component.provider.ComponentRegistry +import com.yahoo.jdisc.http.filter.chain.{RequestFilterChain, ResponseFilterChain} +import com.yahoo.jdisc.http.filter.{RequestFilter, ResponseFilter} +import com.yahoo.jdisc.http.filter.{SecurityResponseFilterChain, SecurityRequestFilterChain, SecurityResponseFilter, SecurityRequestFilter} +import com.yahoo.component.{ComponentSpecification, ComponentId, AbstractComponent} +import com.yahoo.component.chain.model.ChainsModelBuilder +import com.yahoo.component.chain.{Chain, ChainedComponent, ChainsConfigurer} +import com.yahoo.processing.execution.chain.ChainRegistry +import FilterChainRepository._ +import scala.collection.JavaConversions._ + + +/** + * Creates JDisc request/response filter chains. + * @author tonytv + */ +class FilterChainRepository(chainsConfig: ChainsConfig, + requestFilters: ComponentRegistry[RequestFilter], + responseFilters: ComponentRegistry[ResponseFilter], + securityRequestFilters: ComponentRegistry[SecurityRequestFilter], + securityResponseFilters: ComponentRegistry[SecurityResponseFilter]) extends AbstractComponent { + + private val filtersAndChains = new ComponentRegistry[AnyRef] + addAllFilters(filtersAndChains, requestFilters, responseFilters, securityRequestFilters, securityResponseFilters) + addAllChains(filtersAndChains, chainsConfig, requestFilters, responseFilters, securityRequestFilters, securityResponseFilters) + filtersAndChains.freeze() + + def getFilter(componentSpecification: ComponentSpecification) = + filtersAndChains.getComponent(componentSpecification) +} + + private[filter] object FilterChainRepository { + case class FilterWrapper(id: ComponentId, filter: AnyRef) extends ChainedComponent(id) { + def filterType: Class[_] = filter match { + case f: RequestFilter => classOf[RequestFilter] + case f: ResponseFilter => classOf[ResponseFilter] + case f: SecurityRequestFilter => classOf[SecurityRequestFilter] + case f: SecurityResponseFilter => classOf[SecurityResponseFilter] + case _ => throw new IllegalArgumentException("Unsupported filter type: " + filter.getClass.getName) + } + } + + def allFiltersWrapped(registries: ComponentRegistry[_ <: AnyRef]*): ComponentRegistry[FilterWrapper] = { + val wrappedFilters = new ComponentRegistry[FilterWrapper] + + def registerWrappedFilters(registry: ComponentRegistry[_ <: AnyRef]) { + for ((id, filter) <- registry.allComponentsById()) + wrappedFilters.register(id, new FilterWrapper(id, filter)) + } + + registries.foreach(registerWrappedFilters) + wrappedFilters.freeze() + wrappedFilters + } + + private def addAllFilters(destination: ComponentRegistry[AnyRef], registries: ComponentRegistry[_ <: AnyRef]*) { + def wrapSecurityFilter(filter: AnyRef) = { + if (isSecurityFilter(filter)) createSecurityChain(List(filter)) + else filter + } + + for { + registry <- registries + (id, filter) <- registry.allComponentsById() + } destination.register(id, wrapSecurityFilter(filter)) + } + + private def addAllChains(destination: ComponentRegistry[AnyRef], chainsConfig: ChainsConfig, filters: ComponentRegistry[_ <: AnyRef]*) { + val chainRegistry = buildChainsRegistry(chainsConfig, filters) + + for (chain <- chainRegistry.allComponents()) { + destination.register(chain.getId, toJDiscChain(chain)) + } + } + + + def buildChainsRegistry(chainsConfig: ChainsConfig, filters: Seq[ComponentRegistry[_ <: AnyRef]]) = { + val chainRegistry = new ChainRegistry[FilterWrapper] + val chainsModel = ChainsModelBuilder.buildFromConfig(chainsConfig) + + ChainsConfigurer.prepareChainRegistry(chainRegistry, chainsModel, allFiltersWrapped(filters: _*)) + chainRegistry.freeze() + chainRegistry + } + + private def toJDiscChain(chain: Chain[FilterWrapper]): AnyRef = { + checkFilterTypesCompatible(chain) + val jDiscFilters = chain.components() map {_.filter} + + wrapJDiscChain(wrapSecurityFilters(jDiscFilters.toList)) + } + + def wrapJDiscChain(filters: List[AnyRef]): AnyRef = { + if (filters.size == 1) filters.head + else { + filters.head match { + case _: RequestFilter => RequestFilterChain.newInstance(filters.asInstanceOf[List[RequestFilter]]) + case _: ResponseFilter => ResponseFilterChain.newInstance(filters.asInstanceOf[List[ResponseFilter]]) + } + } + } + + def wrapSecurityFilters(filters: List[AnyRef]): List[AnyRef] = { + if (filters.isEmpty) List() + else { + val (securityFilters, rest) = filters.span(isSecurityFilter) + if (securityFilters.isEmpty) { + val (regularFilters, rest) = filters.span(!isSecurityFilter(_)) + regularFilters ++ wrapSecurityFilters(rest) + } else { + createSecurityChain(securityFilters) :: wrapSecurityFilters(rest) + } + } + } + + def createSecurityChain(filters: List[AnyRef]): AnyRef = { + filters.head match { + case _: SecurityRequestFilter => SecurityRequestFilterChain.newInstance(filters.asInstanceOf[List[SecurityRequestFilter]]) + case _: SecurityResponseFilter => SecurityResponseFilterChain.newInstance(filters.asInstanceOf[List[SecurityResponseFilter]]) + case _ => throw new IllegalArgumentException("Unexpected class " + filters.head.getClass) + } + } + + def isSecurityFilter(filter: AnyRef) = { + filter match { + case _: SecurityRequestFilter => true + case _: SecurityResponseFilter => true + case _ => false + } + } + + def checkFilterTypesCompatible(chain: Chain[FilterWrapper]) { + val requestFilters = Set[Class[_]](classOf[RequestFilter], classOf[SecurityRequestFilter]) + val responseFilters = Set[Class[_]](classOf[ResponseFilter], classOf[SecurityResponseFilter]) + + def check(a: FilterWrapper, b: FilterWrapper) { + if (requestFilters(a.filterType) && responseFilters(b.filterType)) + throw new RuntimeException("Can't mix request and response filters in chain %s: %s, %s".format(chain.getId, a.getId, b.getId)) + } + + overlappingPairIterator(chain.components).foreach { + case Seq(_) => + case Seq(filter1: FilterWrapper, filter2: FilterWrapper) => + check(filter1, filter2) + check(filter2, filter1) + } + } + + def overlappingPairIterator[T](s: Seq[T]) = s.iterator.sliding(2, 1) +} |