diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /application/src |
Publish
Diffstat (limited to 'application/src')
57 files changed, 4645 insertions, 0 deletions
diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java new file mode 100644 index 00000000000..9654a798353 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -0,0 +1,661 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application; + +import com.google.common.annotations.Beta; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.impl.StandaloneContainerRunner; +import com.yahoo.application.content.ContentCluster; +import com.yahoo.config.*; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.service.ClientProvider; +import com.yahoo.jdisc.service.ServerProvider; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.Renderer; +import com.yahoo.text.StringUtilities; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.model.VespaModel; +import org.jboss.netty.channel.ChannelException; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.net.BindException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +/** + * Contains one or more containers built from services.xml. + * Other services present in the services.xml file might be mocked in future versions. + * <p> + * Currently, only a single top level JDisc Container is allowed. Other clusters are ignored. + * + * @author tonytv + */ +@Beta +public final class Application implements AutoCloseable { + + private final JDisc container; + private final List<ContentCluster> contentClusters; + private final Path path; + private final boolean deletePathWhenClosing; + + // For internal use only + Application(Path path, Networking networking, boolean deletePathWhenClosing) { + this.path = path; + this.deletePathWhenClosing = deletePathWhenClosing; + contentClusters = ContentCluster.fromPath(path); + container = JDisc.fromPath(path, networking, createVespaModel().configModelRepo()); + } + + @Beta + public static Application fromBuilder(Builder builder) throws Exception { + return builder.build(); + } + + /** + * Factory method to create an Application from an XML String. Note that any components that are referenced in + * the XML must be present on the classpath. To deploy OSGi bundles in memory, + * use {@link Application#fromApplicationPackage(Path, Networking)}. + * + * @param xml the XML configuration to use + * @return a new JDisc instance + */ + public static Application fromServicesXml(String xml, Networking networking) { + Path applicationDir = StandaloneContainerRunner.createApplicationPackage(xml); + return new Application(applicationDir, networking, true); + } + + /** + * Factory method to create an Application from an application package. + * This method allows deploying OSGi bundles(contained in the components subdirectory). + * All the OSGi bundles will share the same class loader. + * + * @param path the reference to the application package to use + * @return a new JDisc instance + */ + public static Application fromApplicationPackage(Path path, Networking networking) { + return new Application(path, networking, false); + } + + /** + * Factory method to create an Application from an application package. + * This method allows deploying OSGi bundles(contained in the components subdirectory). + * All the OSGi bundles will share the same class loader. + * + * @param file the reference to the application package to use + * @return a new JDisc instance + */ + public static Application fromApplicationPackage(File file, Networking networking) { + return fromApplicationPackage(file.toPath(), networking); + } + + private VespaModel createVespaModel() { + try { + DeployState deployState = new DeployState.Builder() + .applicationPackage(FilesApplicationPackage.fromFile(path.toFile())) + .deployLogger((level, s) -> { }) + .build(); + return new VespaModel(new NullConfigModelRegistry(), deployState); + } catch (IOException | SAXException e) { + throw new IllegalArgumentException("Error creating application from '" + path + "'", e); + } + } + + /** + * @param id from the jdisc element in services xml. Default id in services.xml is "jdisc" + */ + public JDisc getJDisc(String id) { + return container; + } + + /** + * Shuts down all services. + */ + @Override + public void close() { + container.close(); + if (deletePathWhenClosing) + IOUtils.recursiveDeleteDir(path.toFile()); + } + + /** + * A wrapper around ApplicationBuilder that generates a services.xml + */ + @Beta + public static class Builder { + private static final ThreadLocal<Random> random = new ThreadLocal<>(); + private static final String DEFAULT_CHAIN = "default"; + + private final Map<String, Container> containers = new LinkedHashMap<>(); + private final Path path; + private Networking networking = Networking.disable; + + public Builder() throws IOException { + this.path = makeTempDir("app", "standalone").toPath(); + } + + public Builder container(String id, Container container) { + if (containers.size() > 0) { + throw new RuntimeException("Only a single JDisc container is currently supported."); + } + containers.put(id, container); + return this; + } + + /** + * Create a temporary directory using @{link File.createTempFile()}, but creating + * a directory instead of a file. + * + * @param prefix directory prefix + * @param suffix directory suffix + * @return The created directory + * @throws IOException if the temporary directory could not be created + */ + private static File makeTempDir(String prefix, String suffix) throws IOException { + File tmpDir = File.createTempFile(prefix, suffix, getTempDir()); + if (!tmpDir.delete()) { + throw new RuntimeException("Could not delete temp directory: " + tmpDir); + } + if (!tmpDir.mkdirs()) { + throw new RuntimeException("Could not create temp directory: " + tmpDir); + } + return tmpDir; + } + + /** + * Get the temporary directory + * + * @return The temporary directory File object + */ + private static File getTempDir() { + String rootPath = getResourceFile("/"); + String tmpPath = rootPath + "/tmp/"; + File tmpDir = new File(tmpPath); + if (!tmpDir.exists() && !tmpDir.mkdirs()) { + if (!tmpDir.exists()) { // possible race condition may cause mkdirs() to fail, check a second time before failing + throw new RuntimeException("Could not create temp dir: " + tmpDir.getAbsolutePath()); + } + } + if (!tmpDir.isDirectory()) { + throw new RuntimeException("Temp dir path is not a directory: " + tmpDir.getAbsolutePath()); + } + return tmpDir; + } + + /** + * Get the file name (path) of a resource or fail if it can not be found + * + * @param resource Name of desired resource + * @return Path of resource + */ + private static String getResourceFile(String resource) { + URL resourceUrl = Application.class.getResource(resource); + if (resourceUrl == null || resourceUrl.getFile() == null || resourceUrl.getFile().isEmpty()) { + throw new RuntimeException("Could not access resource: " + resource); + } + + return resourceUrl.getFile(); + } + + // copy from com.yahoo.application.ApplicationBuilder + private void createFile(final Path path, final String content) throws IOException { + Files.createDirectories(path.getParent()); + Files.write(path, Utf8.toBytes(content)); + } + + /** + * @return a random number between 2000 and 62000 + */ + private static int getRandomPort() { + Random r = random.get(); + if (r == null) { + r = new Random(System.currentTimeMillis()); + random.set(r); + } + return r.nextInt(60000) + 2000; + } + + /** + * @param name name of document type (search definition) + * @param searchDefinition add this search definition to the application + * @return builder + * @throws java.io.IOException e.g.if file not found + */ + public Builder documentType(final String name, final String searchDefinition) throws IOException { + Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX); + createFile(path, searchDefinition); + return this; + } + + public Builder expressionInclude(final String name, final String searchDefinition) throws IOException { + Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); + createFile(path, searchDefinition); + return this; + } + + /** + * @param name name of rank expression + * @param rankExpressionContent add this rank expression to the application + * @return builder + * @throws java.io.IOException e.g.if file not found + */ + public Builder rankExpression(final String name, final String rankExpressionContent) throws IOException { + Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); + createFile(path, rankExpressionContent); + return this; + } + + /** + * @param name name of query profile + * @param queryProfile add this queyr profile to the application + * @return builder + * @throws java.io.IOException e.g.if file not found + */ + public Builder queryProfile(final String name, final String queryProfile) throws IOException { + Path path = nestedResource(ApplicationPackage.QUERY_PROFILES_DIR, name, ".xml"); + createFile(path, queryProfile); + return this; + } + + /** + * @param name name of query profile type + * @param queryProfileType add this query profile type to the application + * @return builder + * @throws java.io.IOException e.g.if file not found + */ + public Builder queryProfileType(final String name, final String queryProfileType) throws IOException { + Path path = nestedResource(ApplicationPackage.QUERY_PROFILE_TYPES_DIR, name, ".xml"); + createFile(path, queryProfileType); + return this; + } + + // copy from com.yahoo.application.ApplicationBuilder + private Path nestedResource(final com.yahoo.path.Path nestedPath, final String name, final String fileType) { + String nameWithoutSuffix = StringUtilities.stripSuffix(name, fileType); + return path.resolve(nestedPath.getRelative()).resolve(nameWithoutSuffix + fileType); + } + + /** + * @param networking enable or disable networking (disabled by default) + * @return builder + */ + public Builder networking(final Networking networking) { + this.networking = networking; + return this; + } + + // generate the services xml and load the container + private Application build() throws Exception { + Application app = null; + Exception exception = null; + + // if we get a bind exception, then retry a few times (may conflict with parallel test runs) + for (int i = 0; i < 5; i++) { + try { + generateXml(); + app = new Application(path, networking, true); + break; + } catch (Error e) { // the container thinks this is really serious, in this case is it not in the cause is a BindException + // catch bind error and reset container + if (e.getCause() != null && e.getCause() instanceof ChannelException && e.getCause().getCause() != null && e.getCause().getCause() instanceof BindException) { + exception = (Exception) e.getCause().getCause(); + com.yahoo.container.Container.resetInstance(); // this is needed to be able to recreate the container from config again + } else { + throw new Exception(e.getCause()); + } + } + } + + if (app == null) { + throw exception; + } + return app; + } + + private void generateXml() throws Exception { + try (PrintWriter xml = new PrintWriter(Files.newOutputStream(path.resolve("services.xml")))) { + xml.println("<?xml version=\"1.0\" encoding=\"utf-8\" ?>"); + //xml.println("<services version=\"1.0\">"); + for (Map.Entry<String, Container> entry : containers.entrySet()) { + entry.getValue().build(xml, entry.getKey(), (networking == Networking.enable ? getRandomPort() : -1)); + } + //xml.println("</services>"); + } + } + + public static class Container { + private final Map<String, List<ComponentItem<? extends DocumentProcessor>>> docprocs = new LinkedHashMap<>(); + private final Map<String, List<ComponentItem<? extends Searcher>>> searchers = new LinkedHashMap<>(); + private final List<ComponentItem<? extends Renderer>> renderers = new ArrayList<>(); + private final List<ComponentItem<? extends RequestHandler>> handlers = new ArrayList<>(); + private final List<ComponentItem<? extends ClientProvider>> clients = new ArrayList<>(); + private final List<ComponentItem<? extends ServerProvider>> servers = new ArrayList<>(); + private final List<ComponentItem<?>> components = new ArrayList<>(); + private final List<ConfigInstance> configs = new ArrayList<>(); + private boolean enableSearch = false; + + private static class ComponentItem<T> { + private String id; + private Class<? extends T> component; + private List<ConfigInstance> configs = new ArrayList<>(); + + public ComponentItem(String id, Class<? extends T> component, ConfigInstance... configs) { + this.id = id; + this.component = component; + if (configs != null) { + Collections.addAll(this.configs, configs); + } + } + } + + /** + * @param docproc add this docproc to the default document processing chain + * @return builder + */ + public Container documentProcessor(final Class<? extends DocumentProcessor> docproc) { + return documentProcessor(DEFAULT_CHAIN, docproc); + } + + /** + * @param chainName chain name to add docproc + * @param docproc add this docproc to the document processing chain + * @param configs local docproc configs + * @return builder + */ + public Container documentProcessor(final String chainName, final Class<? extends DocumentProcessor> docproc, ConfigInstance... configs) { + return documentProcessor(docproc.getName(), chainName, docproc, configs); + } + + /** + * @param id component id + * @param chainName chain name to add docproc + * @param docproc add this docproc to the document processing chain + * @param configs local docproc configs + * @return builder + */ + public Container documentProcessor(String id, final String chainName, final Class<? extends DocumentProcessor> docproc, ConfigInstance... configs) { + List<ComponentItem<? extends DocumentProcessor>> chain = docprocs.get(chainName); + if (chain == null) { + chain = new ArrayList<>(); + docprocs.put(chainName, chain); + } + chain.add(new ComponentItem<>(id, docproc, configs)); + return this; + } + + /** + * @param enableSearch if true, enable search even without any searchers defined + * @return builder + */ + public Container search(boolean enableSearch) { + this.enableSearch = enableSearch; + return this; + } + + /** + * @param searcher add this searcher to the default search chain + * @return builder + */ + public Container searcher(final Class<? extends Searcher> searcher) { + return searcher(DEFAULT_CHAIN, searcher); + } + + /** + * @param chainName chain name to add searcher + * @param searcher add this searcher to the search chain + * @param configs local searcher configs + * @return builder + */ + public Container searcher(final String chainName, final Class<? extends Searcher> searcher, ConfigInstance... configs) { + return searcher(searcher.getName(), chainName, searcher, configs); + } + + /** + * @param id component id + * @param chainName chain name to add searcher + * @param searcher add this searcher to the search chain + * @param configs local searcher configs + * @return builder + */ + public Container searcher(String id, final String chainName, final Class<? extends Searcher> searcher, ConfigInstance... configs) { + List<ComponentItem<? extends Searcher>> chain = searchers.get(chainName); + if (chain == null) { + chain = new ArrayList<>(); + searchers.put(chainName, chain); + } + chain.add(new ComponentItem<>(id, searcher, configs)); + return this; + } + + /** + * @param id component id, enable template with ?format=id or ?presentation.format=id + * @param renderer add this renderer + * @param configs local renderer configs + * @return builder + */ + public Container renderer(String id, final Class<? extends Renderer> renderer, ConfigInstance... configs) { + renderers.add(new ComponentItem<>(id, renderer, configs)); + return this; + } + + /** + * @param binding binding string + * @param handler the handler class + * @return builder + */ + public Container handler(final String binding, final Class<? extends RequestHandler> handler) { + handlers.add(new ComponentItem<>(binding, handler)); + return this; + } + + /** + * @param binding binding string + * @param client the client class + * @return builder + */ + public Container client(final String binding, final Class<? extends ClientProvider> client) { + clients.add(new ComponentItem<>(binding, client)); + return this; + } + + /** + * @param id server compoent id + * @param server the server class + * @return builder + */ + public Container server(final String id, final Class<? extends ServerProvider> server) { + servers.add(new ComponentItem<>(id, server)); + return this; + } + + /** + * @param component make this component available to the container + * @return builder + */ + public Container component(final Class<?> component) { + return component(component.getName(), component, (ConfigInstance) null); + } + + /** + * @param component make this component available to the container + * @return builder + */ + public Container component(String id, final Class<?> component, ConfigInstance... configs) { + components.add(new ComponentItem<>(id, component, configs)); + return this; + } + + /** + * @param config add this config to the application + * @return builder + */ + public Container config(final ConfigInstance config) { + configs.add(config); + return this; + } + + // generate services.xml based on this builder + private void build(PrintWriter xml, String id, int port) throws Exception { + xml.println("<jdisc version=\"1.0\" id=\"" + id + "\">"); + + if (port > 0) { + xml.println("<http>"); + xml.println("<server id=\"http\" port=\"" + port + "\" />"); + xml.println("</http>"); + } + + for (ComponentItem<? extends RequestHandler> entry : handlers) { + xml.println("<handler id=\"" + entry.component.getName() + "\">"); + xml.println("<binding>" + entry.id + "</binding>"); + xml.println("</handler>"); + } + + for (ComponentItem<? extends ClientProvider> entry : clients) { + xml.println("<client id=\"" + entry.component.getName() + "\">"); + xml.println("<binding>" + entry.id + "</binding>"); + xml.println("</client>"); + } + + for (ComponentItem<? extends ServerProvider> server : servers) { + generateComponent(xml, server, "server"); + } + + // container scoped configs + for (ConfigInstance config : configs) { + generateConfig(xml, config); + } + + for (ComponentItem<?> component : components) { + generateComponent(xml, component, "component"); + } + + if (!docprocs.isEmpty()) { + xml.println("<document-processing>"); + for (Map.Entry<String, List<ComponentItem<? extends DocumentProcessor>>> entry : docprocs.entrySet()) { + xml.println("<chain id=\"" + entry.getKey() + "\">"); + for (ComponentItem<? extends DocumentProcessor> docproc : entry.getValue()) { + generateComponent(xml, docproc, "documentprocessor"); + } + xml.println("</chain>"); + } + xml.println("</document-processing>"); + } + + if (enableSearch || !searchers.isEmpty() || !renderers.isEmpty()) { + xml.println("<search>"); + for (Map.Entry<String, List<ComponentItem<? extends Searcher>>> entry : searchers.entrySet()) { + xml.println("<chain id=\"" + entry.getKey() + "\">"); + for (ComponentItem<? extends Searcher> searcher : entry.getValue()) { + generateComponent(xml, searcher, "searcher"); + } + xml.println("</chain>"); + } + for (ComponentItem<? extends Renderer> renderer : renderers) { + generateComponent(xml, renderer, "renderer"); + } + xml.println("</search>"); + } + + xml.println("</jdisc>"); + } + + private void generateComponent(PrintWriter xml, ComponentItem<?> componentItem, String elementName) throws Exception { + xml.print("<" + elementName + " id=\"" + componentItem.id + "\" class=\"" + componentItem.component.getName() + "\""); + if (componentItem.configs.isEmpty() || (!componentItem.configs.isEmpty() && componentItem.configs.get(0) == null)) { + xml.println(" />"); + } else { + xml.println(">"); + for (ConfigInstance config : componentItem.configs) { + generateConfig(xml, config); + } + xml.println("</" + elementName + ">"); + } + } + + // uses reflection to generate XML from a config object + private void generateConfig(PrintWriter xml, ConfigInstance config) throws Exception { + Field nameField = config.getClass().getField("CONFIG_DEF_NAME"); + String name = (String) nameField.get(config); + Field namespaceField = config.getClass().getField("CONFIG_DEF_NAMESPACE"); + String namespace = (String) namespaceField.get(config); + + xml.println("<config name=\"" + namespace + "." + name + "\">"); + generateConfigNode(xml, config); + xml.println("</config>"); + } + + private void generateConfigNode(PrintWriter xml, InnerNode node) throws Exception { + // print all leaf nodes as config values + Field[] fields = node.getClass().getDeclaredFields(); + for (Field field : fields) { + generateConfigField(xml, node, field); + } + } + + private void generateConfigField(PrintWriter xml, InnerNode node, Field field) throws Exception { + field.setAccessible(true); + if (LeafNode.class.isAssignableFrom(field.getType())) { + LeafNode<?> value = (LeafNode<?>) field.get(node); + if (value.value() != null) { + xml.print("<" + field.getName()); + String v = value.getValue(); + if (v.isEmpty()) { + xml.println(" />"); + } else { + xml.println(">" + v + "</" + field.getName() + ">"); + } + } + } else if (InnerNode.class.isAssignableFrom(field.getType())) { + xml.println("<" + field.getName() + ">"); + generateConfigNode(xml, (InnerNode) field.get(node)); + xml.println("</" + field.getName() + ">"); + } else if (Map.class.isAssignableFrom(field.getType())) { + Map<?, ?> map = (Map<?, ?>) field.get(node); + if (!map.isEmpty()) { + xml.println("<" + field.getName() + ">"); + for (Map.Entry<?, ?> entry : map.entrySet()) { + if (entry.getValue() instanceof InnerNode) { + xml.println("<item key=\"" + entry.getKey() + "\">"); + generateConfigNode(xml, (InnerNode) entry.getValue()); + xml.println("</item>"); + } else if (entry.getValue() instanceof LeafNode) { + xml.println("<item key=\"" + entry.getKey() + "\">" + ((LeafNode<?>) entry.getValue()).getValue() + "</item>"); + } + } + xml.println("</" + field.getName() + ">"); + } + } else if (InnerNodeVector.class.isAssignableFrom(field.getType())) { + InnerNodeVector<? extends InnerNode> vector = (InnerNodeVector<? extends InnerNode>) field.get(node); + if (!vector.isEmpty()) { + xml.println("<" + field.getName() + ">"); + for (InnerNode innerNode : vector) { + xml.println("<item>"); + generateConfigNode(xml, innerNode); + xml.println("</item>"); + } + xml.println("</" + field.getName() + ">"); + } + } else if (LeafNodeVector.class.isAssignableFrom(field.getType())) { + LeafNodeVector<?, ? extends LeafNode<?>> vector = (LeafNodeVector<?, ? extends LeafNode<?>>) field.get(node); + if (!vector.isEmpty()) { + xml.println("<" + field.getName() + ">"); + for (LeafNode<?> item : vector) { + xml.println("<item>" + item.getValue() + "</item>"); + } + xml.println("</" + field.getName() + ">"); + } + } + } + } + } +}
\ No newline at end of file diff --git a/application/src/main/java/com/yahoo/application/ApplicationBuilder.java b/application/src/main/java/com/yahoo/application/ApplicationBuilder.java new file mode 100644 index 00000000000..812f29b9f44 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/ApplicationBuilder.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application; + +import com.google.common.annotations.Beta; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.text.StringUtilities; +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static java.nio.file.Files.createTempDirectory; + +/** + * Builds an application package on disk and returns a path to the result. + * + * @author tonytv + */ +@Beta +public class ApplicationBuilder { + private Path applicationDir = createTempDirectory("application"); + private Networking networking = Networking.disable; + + public ApplicationBuilder() throws IOException {} + + public ApplicationBuilder servicesXml(String servicesXml) throws IOException { + ensureNotAlreadyBuild(); + + String content = servicesXml.startsWith("<?xml") ? + servicesXml : + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + '\n' + servicesXml; + + createFile(applicationDir.resolve("services.xml"), content); + return this; + } + + public ApplicationBuilder documentType(String name, String searchDefinition) throws IOException { + Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX); + createFile(path, searchDefinition); + return this; + } + + public ApplicationBuilder rankExpression(String name, String rankExpressionContent) throws IOException { + Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); + createFile(path, rankExpressionContent); + return this; + } + + public ApplicationBuilder queryProfile(String name, String queryProfile) throws IOException { + Path path = nestedResource(ApplicationPackage.QUERY_PROFILES_DIR, name, ".xml"); + createFile(path, queryProfile); + return this; + } + + public ApplicationBuilder queryProfileType(String name, String queryProfileType) throws IOException { + Path path = nestedResource(ApplicationPackage.QUERY_PROFILE_TYPES_DIR, name, ".xml"); + createFile(path, queryProfileType); + return this; + } + + /** + * Disabled per default + */ + public ApplicationBuilder networking(Networking networking) { + this.networking = networking; + return this; + } + + public Application build() { + Application application = new Application(applicationDir, networking, true); + applicationDir = null; + return application; + } + + private Path nestedResource(com.yahoo.path.Path nestedPath, String name, String fileType) { + ensureNotAlreadyBuild(); + + String nameWithoutSuffix = StringUtilities.stripSuffix(name, fileType); + return applicationDir.resolve(nestedPath.getRelative()).resolve(nameWithoutSuffix + fileType); + } + + private void ensureNotAlreadyBuild() { + if (applicationDir == null) + throw new RuntimeException("The ApplicationBuilder must not be used after the build method has been called."); + } + + private void createFile(Path path, String content) throws IOException { + Files.createDirectories(path.getParent()); + Files.write(path, Utf8.toBytes(content)); + } + + Path getPath() { + return applicationDir; + } +} diff --git a/application/src/main/java/com/yahoo/application/Networking.java b/application/src/main/java/com/yahoo/application/Networking.java new file mode 100644 index 00000000000..cc09256203b --- /dev/null +++ b/application/src/main/java/com/yahoo/application/Networking.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application; + +import com.google.common.annotations.Beta; + +/** + * @author tonytv + */ +public enum Networking { + enable, + disable +} diff --git a/application/src/main/java/com/yahoo/application/container/ApplicationException.java b/application/src/main/java/com/yahoo/application/container/ApplicationException.java new file mode 100644 index 00000000000..175b6dc5fd7 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/ApplicationException.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +/** + * Wraps an Exception in a RuntimeException, for user convenience. + * + * @author gjoranv + * @since 5.1.15 + */ +class ApplicationException extends RuntimeException { + ApplicationException(Exception e) { + super(e); + } +} diff --git a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java new file mode 100644 index 00000000000..d263c1d1591 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.google.common.annotations.Beta; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.docproc.DocprocExecutor; +import com.yahoo.docproc.DocprocService; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.jdisc.DocumentProcessingHandler; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.processing.execution.chain.ChainRegistry; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * For doing document processing with {@link JDisc}. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +@Beta +public final class DocumentProcessing { + private final DocumentProcessingHandler handler; + private final Map<String, DocumentType> documentTypes; + + DocumentProcessing(DocumentProcessingHandler handler) { + this.handler = handler; + documentTypes = retrieveDocumentTypes(handler.getDocumentTypeManager()); + } + + private static Map<String, DocumentType> retrieveDocumentTypes(DocumentTypeManager documentTypeManager) { + Map<String, DocumentType> documentTypes = new HashMap<>() ; + for (Iterator<DocumentType> i = documentTypeManager.documentTypeIterator(); i.hasNext();) { + DocumentType type = i.next(); + documentTypes.put(type.getName(), type); + } + return Collections.unmodifiableMap(documentTypes); + } + + /** + * Processes the given Processing through the specified chain. Note that if one + * {@link com.yahoo.docproc.DocumentProcessor DocumentProcessor} in the + * chain returns a {@link com.yahoo.docproc.DocumentProcessor.LaterProgress DocumentProcessor.LaterProgress}, + * the calling thread will sleep for the duration + * specified in {@link com.yahoo.docproc.DocumentProcessor.LaterProgress#getDelay() DocumentProcessor.LaterProgress#getDelay()}, + * and then run again. This method will hence return when a document processor returns + * {@link com.yahoo.docproc.DocumentProcessor.Progress#DONE DocumentProcessor.Progress#DONE} or + * {@link com.yahoo.docproc.DocumentProcessor.Progress#FAILED DocumentProcessor.Progress#FAILED}, throws an exception, + * or if the calling thread is interrupted. This method will never return a + * {@link com.yahoo.docproc.DocumentProcessor.LaterProgress DocumentProcessor.LaterProgress}. + * + * @param chain the specification of the chain to execute + * @param processing the Processing to process + * @return Progress.DONE or Progress.FAILED + * @throws RuntimeException if one of the document processors in the chain throws, or if the calling thread is interrupted + */ + public DocumentProcessor.Progress process(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { + DocprocExecutor executor = getExecutor(chain); + processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); + return executor.processUntilDone(processing); + } + + /** + * Processes the given Processing through the specified chain. Note that if one + * {@link com.yahoo.docproc.DocumentProcessor DocumentProcessor} in the + * chain returns a {@link com.yahoo.docproc.DocumentProcessor.LaterProgress DocumentProcessor.LaterProgress}, + * it will be returned by this method. This method will hence return whenever a document processor returns any + * {@link com.yahoo.docproc.DocumentProcessor.Progress DocumentProcessor.Progress}, or + * throws an exception. + * + * @param chain the specification of the chain to execute + * @param processing the Processing to process + * @return any Progress + * @throws RuntimeException if one of the document processors in the chain throws + */ + public DocumentProcessor.Progress processOnce(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { + DocprocExecutor executor = getExecutor(chain); + processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); + return executor.process(processing); + } + + private DocprocExecutor getExecutor(ComponentSpecification chain) { + DocprocService service = handler.getDocprocServiceRegistry().getComponent(chain); + if (service == null) { + throw new IllegalArgumentException("No such chain: " + chain); + } + return service.getExecutor(); + } + + /** + * Returns a registry of configured docproc chains. + * + * @return a registry of configured docproc chains + */ + public ChainRegistry<DocumentProcessor> getChains() { + return handler.getChains(); + } + + public Map<String, DocumentType> getDocumentTypes() { + return documentTypes; + } + + public Map<String, AnnotationType> getAnnotationTypes() { + return handler.getDocumentTypeManager().getAnnotationTypeRegistry().getTypes(); + } + +} diff --git a/application/src/main/java/com/yahoo/application/container/JDisc.java b/application/src/main/java/com/yahoo/application/container/JDisc.java new file mode 100644 index 00000000000..d6cc4534972 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/JDisc.java @@ -0,0 +1,206 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.google.common.annotations.Beta; +import com.google.inject.AbstractModule; +import com.google.inject.Module; +import com.google.inject.name.Names; +import com.yahoo.application.Networking; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.container.Container; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.application.container.impl.ClassLoaderOsgiFramework; +import com.yahoo.application.container.impl.StandaloneContainerRunner; +import com.yahoo.container.standalone.StandaloneContainerApplication; +import com.yahoo.container.standalone.StandaloneContainerApplication$; +import com.yahoo.docproc.jdisc.DocumentProcessingHandler; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.test.TestDriver; +import com.yahoo.processing.handler.ProcessingHandler; +import com.yahoo.search.handler.SearchHandler; + +import java.nio.file.Path; + +/** + * A JDisc Container configured from XML. + * + * @author tonytv + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author gjoranv + * @since 5.1.15 + */ +@Beta +public final class JDisc implements AutoCloseable { + + private final ClassLoaderOsgiFramework osgiFramework = new ClassLoaderOsgiFramework(); + + private final TestDriver testDriver; + private final StandaloneContainerApplication application; + + private final Container container = Container.get(); // TODO: This is indeed temporary ... *3 years later* Indeed. + + private final Path path; + private final boolean deletePathWhenClosing; + + private JDisc(Path path, boolean deletePathWhenClosing, Networking networking, ConfigModelRepo configModelRepo) { + try { + this.path = path; + this.deletePathWhenClosing = deletePathWhenClosing; + testDriver = TestDriver.newInstance(osgiFramework, "", false, //StandaloneContainerApplication.class, + bindings(path, configModelRepo, networking)); + + application = (StandaloneContainerApplication) testDriver.application(); + } catch(Throwable t) { + StackTrace.filterLogAndDieToJDiscInit(t); + throw t; + } + } + + private Module bindings(final Path path, final ConfigModelRepo configModelRepo, final Networking networking) { + return new AbstractModule() { + @Override + protected void configure() { + bind(Path.class).annotatedWith(StandaloneContainerApplication.applicationPathName()).toInstance(path); + bind(ConfigModelRepo.class).annotatedWith(StandaloneContainerApplication.configModelRepoName()).toInstance(configModelRepo); + bind(Boolean.class).annotatedWith( // below is an ugly hack to access fields from a scala object. + Names.named(StandaloneContainerApplication$.MODULE$.disableNetworkingAnnotation())).toInstance( + networking == Networking.disable); + } + }; + } + + /** + * Factory method to create a JDisc from an XML String. Note that any components that are referenced in + * the XML must be present on the classpath. To deploy OSGi bundles in memory, + * use {@link #fromPath(java.nio.file.Path, com.yahoo.application.Networking)}. + * + * @param xml the XML configuration to use + * @return a new JDisc instance + */ + public static JDisc fromServicesXml(String xml, Networking networking) { + Path applicationDir = StandaloneContainerRunner.createApplicationPackage(xml); + return new JDisc(applicationDir, true, networking, new ConfigModelRepo()); + } + + /** + * Factory method to create a JDisc from an application package. + * This method allows deploying OSGi bundles(contained in the components subdirectory). + * All the OSGi bundles will share the same class loader. + * + * + * + * @param path the reference to the application package to use + * @param networking enabled or disabled + * @return a new JDisc instance + */ + public static JDisc fromPath(final Path path, Networking networking) { + return new JDisc(path, false, networking, new ConfigModelRepo()); + } + + /** + * Create a jDisc instance which is given a config model repo (in which (mock) content clusters + * can be looked up). + */ + public static JDisc fromPath(final Path path, Networking networking, ConfigModelRepo configModelRepo) { + return new JDisc(path, false, networking, configModelRepo); + } + + /** + * Returns a {@link Search}, used to perform search query operations on this container. + * + * @return a Search instance + * @throws UnsupportedOperationException if this JDisc does not have search configured + */ + public Search search() { + SearchHandler searchHandler = getSearchHandler(); + if (searchHandler == null) + throw new UnsupportedOperationException("This JDisc does not have 'search' " + "configured."); + return new Search(searchHandler); + } + + private SearchHandler getSearchHandler() { + for (RequestHandler h : container.getRequestHandlerRegistry().allComponents()) { + if (h instanceof SearchHandler) { + return (SearchHandler) h; + } + } + return null; + } + + /** + * Returns a {@link Processing}, used to do generic asynchronous operations in a request/response API. + * + * @return a Processing instance + * @throws UnsupportedOperationException if this JDisc does not have processing configured + */ + public Processing processing() { + ProcessingHandler processingHandler = (ProcessingHandler) container + .getRequestHandlerRegistry() + .getComponent(ProcessingHandler.class.getName()); + + if (processingHandler == null) { + throw new UnsupportedOperationException("This JDisc does not have 'processing' " + + "configured."); + } + + return new Processing(processingHandler); + } + + /** + * Returns a {@link DocumentProcessing}, used to process objects of type {@link com.yahoo.document.Document}, + * {@link com.yahoo.document.DocumentRemove} and {@link com.yahoo.document.DocumentUpdate}. + * + * @return a DocumentProcessing instance + * @throws UnsupportedOperationException if this JDisc does not have document processing configured + */ + public DocumentProcessing documentProcessing() { + DocumentProcessingHandler docprocHandler = (DocumentProcessingHandler) container + .getRequestHandlerRegistry() + .getComponent(DocumentProcessingHandler.class.getName()); + + if (docprocHandler == null) { + throw new UnsupportedOperationException("This JDisc does not have 'document-processing' " + + "configured."); + } + return new DocumentProcessing(docprocHandler); + } + + /** + * Returns a registry of all components available in this + */ + public ComponentRegistry<AbstractComponent> components() { + return container.getComponentRegistry(); + } + + /** + * Handles the given {@link com.yahoo.application.container.handler.Request} by passing it to the {@link RequestHandler} + * that is bound to the request's URI. + * + * @param request the request to process + * @return a response for the given request + */ + public Response handleRequest(Request request) { + SynchronousRequestResponseHandler handler = new SynchronousRequestResponseHandler(); + return handler.handleRequest(request, testDriver); + } + + /** + * Closes the current JDisc. + */ + @Override + public void close() { + try { + testDriver.close(); + } finally { + Container.resetInstance(); + + if (deletePathWhenClosing) + IOUtils.recursiveDeleteDir(path.toFile()); + } + } + +} diff --git a/application/src/main/java/com/yahoo/application/container/Processing.java b/application/src/main/java/com/yahoo/application/container/Processing.java new file mode 100644 index 00000000000..1511024364d --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/Processing.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.google.common.annotations.Beta; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Chain; +import com.yahoo.processing.Processor; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.execution.Execution; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.processing.handler.ProcessingHandler; +import com.yahoo.processing.rendering.Renderer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author gjoranv +*/ +@Beta +public final class Processing extends ProcessingBase<Request, Response, Processor> { + private final ProcessingHandler handler; + + Processing(ProcessingHandler handler) { + this.handler = handler; + } + + @Override + public ChainRegistry<Processor> getChains() { + return handler.getChainRegistry(); + } + + @Override + protected Response doProcess(Chain<Processor> chain, Request request) { + Execution execution = handler.createExecution(chain, request); + return execution.process(request); + } + + @Override + protected ListenableFuture<Boolean> doProcessAndRender(ComponentSpecification chainSpec, + Request request, + Renderer<Response> renderer, + ByteArrayOutputStream stream) throws IOException { + Execution execution = handler.createExecution(getChain(chainSpec), request); + Response response = execution.process(request); + + return renderer.render(stream, response, execution, request); + } + + @Override + protected Renderer<Response> doGetRenderer(ComponentSpecification spec) { + return handler.getRendererCopy(spec); + } + +} diff --git a/application/src/main/java/com/yahoo/application/container/ProcessingBase.java b/application/src/main/java/com/yahoo/application/container/ProcessingBase.java new file mode 100644 index 00000000000..6637669874d --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/ProcessingBase.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.google.common.annotations.Beta; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Chain; +import com.yahoo.processing.Processor; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.processing.rendering.AsynchronousRenderer; +import com.yahoo.processing.rendering.Renderer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; + +/** + * @author gjoranv + * @since 5.1.15 + */ +@Beta +public abstract class ProcessingBase< + REQUEST extends Request, + RESPONSE extends Response, + PROCESSOR extends Processor> +{ + /** + * Returns a registry of configured chains. + * + * @return a registry of configured chains + */ + public abstract ChainRegistry<PROCESSOR> getChains(); + + /** + * Processes the given request with the given chain, and returns the response. + * + * @param chain the specification of the chain to execute + * @param request the request to process + * @return a response + */ + public final RESPONSE process(ComponentSpecification chain, REQUEST request) { + Chain<PROCESSOR> processingChain = getChain(chain); + return doProcess(processingChain, request); + } + + protected abstract RESPONSE doProcess(Chain<PROCESSOR> chain, REQUEST request); + + public final byte[] processAndRender(ComponentSpecification chainSpec, + ComponentSpecification rendererSpec, + REQUEST request) throws IOException { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + Renderer<RESPONSE> renderer = getRenderer(rendererSpec); + ListenableFuture<Boolean> renderTask = doProcessAndRender(chainSpec, request, renderer, stream); + + awaitFuture(renderTask); + return stream.toByteArray(); + } + + private void awaitFuture(ListenableFuture<Boolean> renderTask) { + try { + renderTask.get(); + } catch (InterruptedException | ExecutionException e) { + throw new ApplicationException(e); + } + } + + protected abstract ListenableFuture<Boolean> doProcessAndRender(ComponentSpecification chainSpec, + REQUEST request, + Renderer<RESPONSE> renderer, + ByteArrayOutputStream stream) throws IOException ; + + protected Chain<PROCESSOR> getChain(ComponentSpecification chainSpec) { + Chain<PROCESSOR> chain = getChains().getComponent(chainSpec); + if (chain == null) { + throw new IllegalArgumentException("No such chain: " + chainSpec); + } + return chain; + } + + protected final Renderer<RESPONSE> getRenderer(ComponentSpecification spec) { + return doGetRenderer(spec); + } + + // TODO: This would not be necessary if Search/ProcHandler implemented a common interface + protected abstract Renderer<RESPONSE> doGetRenderer(ComponentSpecification spec); + +} diff --git a/application/src/main/java/com/yahoo/application/container/Search.java b/application/src/main/java/com/yahoo/application/container/Search.java new file mode 100644 index 00000000000..14f1405d40a --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/Search.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.google.common.annotations.Beta; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Chain; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.processing.rendering.Renderer; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.handler.HttpSearchResponse; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.searchchain.SearchChainRegistry; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author gjoranv +*/ +@Beta +public final class Search extends ProcessingBase<Query, Result, Searcher> { + + private final SearchHandler handler; + + Search(SearchHandler handler) { + this.handler = handler; + } + + @Override + public ChainRegistry<Searcher> getChains() { + return asChainRegistry(handler.getSearchChainRegistry()); + } + + @Override + protected Result doProcess(Chain<Searcher> chain, Query request) { + return handler.searchAndFill(request, chain, handler.getSearchChainRegistry()); + } + + @Override + protected ListenableFuture<Boolean> doProcessAndRender(ComponentSpecification chainSpec, + Query request, + Renderer<Result> renderer, + ByteArrayOutputStream stream) throws IOException { + Result result = process(chainSpec, request); + result.getTemplating().setRenderer(renderer); + return HttpSearchResponse.waitableRender(result, result.getQuery(), renderer, stream); + } + + @Override + protected Renderer<Result> doGetRenderer(ComponentSpecification spec) { + return handler.getRendererCopy(spec); + } + + // TODO: move to SearchHandler.getChainRegistry and deprecate SH.getSCReg? + private ChainRegistry<Searcher> asChainRegistry(SearchChainRegistry legacyRegistry) { + ChainRegistry<Searcher> chains = new ChainRegistry<>(); + for (Chain<Searcher> chain : handler.getSearchChainRegistry().allComponents()) + chains.register(chain.getId(), chain); + chains.freeze(); + return chains; + } + +} diff --git a/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java b/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java new file mode 100644 index 00000000000..4702c9fd782 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java @@ -0,0 +1,188 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.google.common.annotations.Beta; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.RequestDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.jdisc.test.TestDriver; + +import javax.annotation.concurrent.ThreadSafe; +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +@ThreadSafe +@Beta +final class SynchronousRequestResponseHandler { + + Response handleRequest(Request request, TestDriver driver) { + BlockingResponseHandler responseHandler = new BlockingResponseHandler(); + ContentChannel inputRequestChannel = connectRequest(request, driver, responseHandler); + writeRequestBody(request, inputRequestChannel); + return responseHandler.getResponse(); + } + + private void writeRequestBody(Request request, ContentChannel inputRequestChannel) { + List<BlockingCompletionHandler> completionHandlers = new ArrayList<>(); + + if (request.getBody().length > 0) { + BlockingCompletionHandler w = new BlockingCompletionHandler(); + try { + inputRequestChannel.write(ByteBuffer.wrap(request.getBody()), w); + completionHandlers.add(w); + } finally { + BlockingCompletionHandler c = new BlockingCompletionHandler(); + inputRequestChannel.close(c); + completionHandlers.add(c); + } + } else { + BlockingCompletionHandler c = new BlockingCompletionHandler(); + inputRequestChannel.close(c); + completionHandlers.add(c); + } + + for (BlockingCompletionHandler completionHandler : completionHandlers) { + completionHandler.waitUntilCompleted(); + } + } + + private ContentChannel connectRequest(final Request request, + final TestDriver driver, + final ResponseHandler responseHandler) { + RequestDispatch dispatch = + new RequestDispatch() { + @Override + protected com.yahoo.jdisc.Request newRequest() { + return createDiscRequest(request, driver); + } + + @Override + public ContentChannel handleResponse(com.yahoo.jdisc.Response response) { + return responseHandler.handleResponse(response); + } + }; + return dispatch.connect(); + } + + private static String getScheme(String uri) { + int colonPos = uri.indexOf(':'); + if (colonPos < 0) { + return ""; + } + return uri.substring(0, colonPos); + } + + + private static com.yahoo.jdisc.Request createDiscRequest(Request request, CurrentContainer currentContainer) { + String scheme = getScheme(request.getUri()); + com.yahoo.jdisc.Request discRequest; + if ("http".equals(scheme) || "https".equals(scheme)) { + discRequest = com.yahoo.jdisc.http.HttpRequest.newServerRequest(currentContainer, + URI.create(request.getUri()), + com.yahoo.jdisc.http.HttpRequest.Method.valueOf(request.getMethod().name())); + } else { + discRequest = new com.yahoo.jdisc.Request(currentContainer, URI.create(request.getUri())); + } + for (Map.Entry<String, List<String>> entry : request.getHeaders().entrySet()) { + discRequest.headers().add(entry.getKey(), entry.getValue()); + } + return discRequest; + } + + private static byte[] concatenateBuffers(List<ByteBuffer> byteBuffers) { + int totalSize = 0; + for (ByteBuffer responseBuffer : byteBuffers) { + totalSize += responseBuffer.remaining(); + } + ByteBuffer totalBuffer = ByteBuffer.allocate(totalSize); + for (ByteBuffer responseBuffer : byteBuffers) { + totalBuffer.put(responseBuffer); + } + return totalBuffer.array(); + } + + private static void copyResponseHeaders(Response response, com.yahoo.jdisc.Response discResponse) { + for (Map.Entry<String, List<String>> entry : discResponse.headers().entrySet()) { + response.getHeaders().put(entry.getKey(), entry.getValue()); + } + } + + @ThreadSafe + private static class BlockingResponseHandler implements ResponseHandler, ContentChannel { + private volatile com.yahoo.jdisc.Response discResponse = null; + private CountDownLatch closedLatch = new CountDownLatch(1); + private final List<ByteBuffer> responseBuffers = new ArrayList<>(); + + @Override + public ContentChannel handleResponse(com.yahoo.jdisc.Response discResponse) { + this.discResponse = discResponse; + return this; + } + + public Response getResponse() { + try { + closedLatch.await(); + } catch (InterruptedException e) { + throw new ApplicationException(e); + } + byte[] totalBuffer = concatenateBuffers(responseBuffers); + Response response = new Response(discResponse.getStatus(), totalBuffer); + copyResponseHeaders(response, discResponse); + return response; + } + + @Override + public void write(ByteBuffer byteBuffer, CompletionHandler completionHandler) { + responseBuffers.add(byteBuffer); + completionHandler.completed(); + } + + @Override + public void close(CompletionHandler completionHandler) { + completionHandler.completed(); + closedLatch.countDown(); + } + } + + private static class BlockingCompletionHandler implements CompletionHandler { + private volatile Throwable throwable; + private CountDownLatch doneLatch = new CountDownLatch(1); + + @Override + public void completed() { + doneLatch.countDown(); + } + + @Override + public void failed(Throwable t) { + throwable = t; + doneLatch.countDown(); + } + + public void waitUntilCompleted() { + try { + doneLatch.await(); + } catch (InterruptedException e) { + throw new ApplicationException(e); + } + if (throwable != null) { + if (throwable instanceof RuntimeException) { + throw (RuntimeException) throwable; + } else { + throw new RuntimeException(throwable); + } + } + } + } +} diff --git a/application/src/main/java/com/yahoo/application/container/handler/Headers.java b/application/src/main/java/com/yahoo/application/container/handler/Headers.java new file mode 100644 index 00000000000..834ce7802f8 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/handler/Headers.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handler; + +import com.google.common.annotations.Beta; +import com.yahoo.jdisc.HeaderFields; + +import javax.annotation.concurrent.NotThreadSafe; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A multi-map for Request and Response header fields. + * + * @see Request + * @see Response + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@NotThreadSafe +@Beta +public class Headers implements Map<String, List<String>> { + private final HeaderFields h = new HeaderFields(); + + @Override + public int size() { + return h.size(); + } + + @Override + public boolean isEmpty() { + return h.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return h.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return h.containsValue(value); + } + + @Override + public List<String> get(Object key) { + return h.get(key); + } + + @Override + public List<String> put(String key, List<String> value) { + return h.put(key, value); + } + + @Override + public List<String> remove(Object key) { + return h.remove(key); + } + + @Override + public void putAll(Map<? extends String, ? extends List<String>> m) { + h.putAll(m); + } + + @Override + public void clear() { + h.clear(); + } + + @Override + public Set<String> keySet() { + return h.keySet(); + } + + @Override + public Collection<List<String>> values() { + return h.values(); + } + + @Override + public Set<Entry<String, List<String>>> entrySet() { + return h.entrySet(); + } + + @Override + public String toString() { + return h.toString(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof Headers && h.equals(((Headers)obj).h); + } + + @Override + public int hashCode() { + return h.hashCode(); + } + + + /* + CONVENIENCE METHODS: + */ + + /** + * <p>Convenience method for checking whether or not a named header contains a specific value. If the named header + * is not set, or if the given value is not contained within that header's value list, this method returns + * <em>false</em>.</p> + * + * <p><em>NOTE:</em> This method is case-SENSITIVE.</p> + * + * @param key The key whose values to search in. + * @param value The values to search for. + * @return True if the given value was found in the named header. + * @see #containsIgnoreCase + */ + public boolean contains(String key, String value) { + return h.contains(key, value); + } + + /** + * <p>Convenience method for checking whether or not a named header contains a specific value, regardless of case. + * If the named header is not set, or if the given value is not contained within that header's value list, this + * method returns <em>false</em>.</p> + * + * <p><em>NOTE:</em> This method is case-INSENSITIVE.</p> + * + * @param key The key whose values to search in. + * @param value The values to search for, ignoring case. + * @return True if the given value was found in the named header. + * @see #contains + */ + public boolean containsIgnoreCase(String key, String value) { + return h.containsIgnoreCase(key, value); + } + + /** + * <p>Adds the given value to the entry of the specified key. If no entry exists for the given key, a new one is + * created containing only the given value.</p> + * + * @param key The key with which the specified value is to be associated. + * @param value The value to be added to the list associated with the specified key. + */ + public void add(String key, String value) { + h.add(key, value); + } + + /** + * <p>Adds the given values to the entry of the specified key. If no entry exists for the given key, a new one is + * created containing only the given values.</p> + * + * @param key The key with which the specified value is to be associated. + * @param values The values to be added to the list associated with the specified key. + */ + public void add(String key, List<String> values) { + h.add(key, values); + } + + /** + * <p>Adds all the entries of the given map to this. This is the same as calling {@link #add(String, List)} for each + * entry in <tt>values</tt>.</p> + * + * @param values The values to be added to this. + */ + public void addAll(Map<? extends String, ? extends List<String>> values) { + h.addAll(values); + } + + /** + * <p>Convenience method to call {@link #put(String, List)} with a singleton list that contains the specified + * value.</p> + * + * @param key The key of the entry to put. + * @param value The value to put. + * @return The previous value associated with <tt>key</tt>, or <tt>null</tt> if there was no mapping for + * <tt>key</tt>. + */ + public List<String> put(String key, String value) { + return h.put(key, value); + } + + /** + * <p>Removes the given value from the entry of the specified key.</p> + * + * @param key The key of the entry to remove from. + * @param value The value to remove from the entry. + * @return True if the value was removed. + */ + public boolean remove(String key, String value) { + return h.remove(key, value); + } + + /** + * <p>Convenience method for retrieving the first value of a named header field. If the header is not set, or if the + * value list is empty, this method returns null.</p> + * + * @param key The key whose first value to return. + * @return The first value of the named header, or null. + */ + public String getFirst(String key) { + return h.getFirst(key); + } + + /** + * <p>Convenience method for checking whether or not a named header field is <em>true</em>. To satisfy this, the + * header field needs to have at least 1 entry, and Boolean.valueOf() of all its values must parse as + * <em>true</em>.</p> + * + * @param key The key whose values to parse as a boolean. + * @return The boolean value of the named header. + */ + public boolean isTrue(String key) { + return h.isTrue(key); + } + + /** + * <p>Returns an unmodifiable list of all key-value pairs of this. This provides a flattened view on the content of + * this map.</p> + * + * @return The collection of entries. + */ + public List<Entry<String, String>> entries() { + return h.entries(); + } +} diff --git a/application/src/main/java/com/yahoo/application/container/handler/Request.java b/application/src/main/java/com/yahoo/application/container/handler/Request.java new file mode 100644 index 00000000000..cd1ac64cb9e --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/handler/Request.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handler; + +import com.google.common.annotations.Beta; +import net.jcip.annotations.Immutable; + +/** + * A request for use with {@link com.yahoo.application.container.JDisc#handleRequest(Request)}. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.15 + * @see Response + */ +@Immutable +@Beta +public class Request { + private final Headers headers = new Headers(); + private final String uri; + private final byte[] body; + private final Method method; + + /** + * Creates a Request with an empty body. + * + * @param uri the URI of the request + */ + public Request(String uri) { + this(uri, new byte[0]); + } + + /** + * Creates a Request with a message body. + * + * @param uri the URI of the request + * @param body the body of the request + */ + public Request(String uri, byte[] body) { + this(uri, body, Method.GET); + } + + + /** + * Creates a Request with a message body. + * + * @param uri the URI of the request + * @param body the body of the request + */ + public Request(String uri, byte[] body, Method method) { + this.uri = uri; + this.body = body; + this.method = method; + } + + /** + * Returns a mutable multi-map of headers for this Request. + * + * @return a mutable multi-map of headers for this Request + */ + public Headers getHeaders() { + return headers; + } + + /** + * Returns the body of this Request. + * + * @return the body of this Request + */ + public byte[] getBody() { + return body; + } + + /** + * Returns the URI of this Request. + * + * @return the URI of this Request + */ + public String getUri() { + return uri; + } + + @Override + public String toString() { + String bodyStr = (body == null || body.length == 0) ? "[empty]" : "[omitted]"; + return "Request to " + uri + ", headers: " + headers + ", body: " + bodyStr; + } + + public Method getMethod() { + return method; + } + + public enum Method { + OPTIONS, + GET, + HEAD, + POST, + PUT, + PATCH, + DELETE, + TRACE, + CONNECT + } +} diff --git a/application/src/main/java/com/yahoo/application/container/handler/Response.java b/application/src/main/java/com/yahoo/application/container/handler/Response.java new file mode 100644 index 00000000000..37450e9d33f --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/handler/Response.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handler; + +import com.google.common.annotations.Beta; +import com.yahoo.jdisc.http.HttpHeaders; +import com.yahoo.text.Utf8; +import net.jcip.annotations.Immutable; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.UnsupportedCharsetException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A response for use with {@link com.yahoo.application.container.JDisc#handleRequest(Request)}. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.15 + * @see Request + */ +@Immutable +@Beta +public class Response { + private final static Pattern charsetPattern = Pattern.compile("charset=([^\\s\\;]+)", Pattern.CASE_INSENSITIVE); + private final int status; + private final Headers headers = new Headers(); + private final byte[] body; + + /** + * Creates a Response with an empty body, and 200 (OK) response code. + */ + public Response() { + this(new byte[0]); + } + + /** + * Creates a Response with a message body, and 200 (OK) response code. + * + * @param body the body of the response + */ + public Response(byte[] body) { + this(com.yahoo.jdisc.Response.Status.OK, body); + } + + /** + * Creates a Response with a message body, and the given response code. + * + * @param status the status code of the response + * @param body the body of the response + * @since 5.1.28 + */ + public Response(int status, byte[] body) { + this.status = status; + this.body = body; + } + + /** + * <p>Returns the status code of this response. This is an integer result code of the attempt to understand and + * satisfy the corresponding {@link com.yahoo.application.container.handler.Request}. + * + * @return The status code. + * @since 5.1.28 + */ + public int getStatus() { + return status; + } + + /** + * Returns the body of this Response. + * + * @return the body of this Response + */ + public byte[] getBody() { + return body; + } + + /** + * Attempts to decode the buffer returned by {@link #getBody()} as a String in a best-effort manner. This is done + * using the Content-Type header - and defaults to UTF-8 encoding if the header is unparseable or not found. + * Note that this may very well throw a {@link CharacterCodingException}. + * + * @return a String with the decoded contents of the body buffer + * @throws CharacterCodingException if the body buffer was not well-formed + */ + public String getBodyAsString() throws CharacterCodingException { + CharsetDecoder decoder = charset().newDecoder(); + return decoder.decode(ByteBuffer.wrap(body)).toString(); + } + + /** + * Returns a mutable multi-map of headers for this Response. + * + * @return a mutable multi-map of headers for this Response + */ + public Headers getHeaders() { + return headers; + } + + @Override + public String toString() { + String bodyStr = (body == null || body.length == 0) ? "[empty]" : "[omitted]"; + return "Response, headers: " + headers + ", body: " + bodyStr; + } + + private Charset charset() { + return charset(headers.getFirst(HttpHeaders.Names.CONTENT_TYPE)); + } + + static Charset charset(String contentType) { + if (contentType != null) { + Matcher matcher = charsetPattern.matcher(contentType); + if (matcher.find()) { + try { + return Charset.forName(matcher.group(1)); + } catch (UnsupportedCharsetException uce) { + return Utf8.getCharset(); + } + } + } + return Utf8.getCharset(); + } +} diff --git a/application/src/main/java/com/yahoo/application/container/handler/package-info.java b/application/src/main/java/com/yahoo/application/container/handler/package-info.java new file mode 100644 index 00000000000..7ae19acccdf --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/handler/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * API for passing requests and inspecting responses to a locally instantiated container. + */ +@ExportPackage +@PublicApi +package com.yahoo.application.container.handler; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/application/src/main/java/com/yahoo/application/container/package-info.java b/application/src/main/java/com/yahoo/application/container/package-info.java new file mode 100644 index 00000000000..2b3a4156ba2 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/container/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * API for interacting with a locally instantiated jDisc container. + */ +@ExportPackage +@PublicApi +package com.yahoo.application.container; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/application/src/main/java/com/yahoo/application/content/ContentCluster.java b/application/src/main/java/com/yahoo/application/content/ContentCluster.java new file mode 100644 index 00000000000..8d1d7530650 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/content/ContentCluster.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.content; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.vespa.model.builder.xml.dom.DomContentBuilder; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ContentCluster { + + /** + * Returns mock of the content clusters described in the application at the given path + * + * @param path the path to an application package + * @return a mock content cluster + */ + public static List<ContentCluster> fromPath(Path path) { + // new DomContentBuilder(). + // TODO + return Collections.<ContentCluster>emptyList(); + } + + /* + public ConfigModel toConfigModel() { + ConfigModel configModel = new ConfigModel(); + return configModel; + } + */ + +} diff --git a/application/src/main/java/com/yahoo/application/package-info.java b/application/src/main/java/com/yahoo/application/package-info.java new file mode 100644 index 00000000000..f374659f435 --- /dev/null +++ b/application/src/main/java/com/yahoo/application/package-info.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * This package provides an API for building Vespa or jDisc applications programmatically or from + * an application package source, and instantiating those applications inside the current Java runtime. + * Currently only a single jDisc cluster and no content clusters are supported. + */ +@ExportPackage +@PublicApi +package com.yahoo.application; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/application/src/main/scala/com/yahoo/application/container/StackTrace.scala b/application/src/main/scala/com/yahoo/application/container/StackTrace.scala new file mode 100644 index 00000000000..09cdb569028 --- /dev/null +++ b/application/src/main/scala/com/yahoo/application/container/StackTrace.scala @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container + +/** + * @author tonytv + */ +object StackTrace { + private val jdiscContainerStartMarker = new StackTraceElement("============= JDiscContainer =============", "start", null, -1) + + def filterLogAndDieToJDiscInit(throwable: Throwable) { + val stackTrace = throwable.getStackTrace + + for { + top <- stackTrace.headOption + if !preserveStackTrace + if top.getClassName == classOf[com.yahoo.protect.Process].getName + index = stackTrace.lastIndexWhere(_.getClassName == classOf[JDisc].getName) + if (index != -1) + } throwable.setStackTrace(jdiscContainerStartMarker +: stackTrace.drop(index)) + } + + //TODO: combine with version in container/di + val preserveStackTrace: Boolean = Option(System.getProperty("jdisc.container.preserveStackTrace")).filterNot(_.isEmpty).isDefined +} diff --git a/application/src/test/app-packages/searcher-app/components/com.yahoo.vespatest.ExtraHitSearcher.jar b/application/src/test/app-packages/searcher-app/components/com.yahoo.vespatest.ExtraHitSearcher.jar Binary files differnew file mode 100644 index 00000000000..90845ab51f9 --- /dev/null +++ b/application/src/test/app-packages/searcher-app/components/com.yahoo.vespatest.ExtraHitSearcher.jar diff --git a/application/src/test/app-packages/searcher-app/services.xml b/application/src/test/app-packages/searcher-app/services.xml new file mode 100644 index 00000000000..f99fbb51621 --- /dev/null +++ b/application/src/test/app-packages/searcher-app/services.xml @@ -0,0 +1,15 @@ +<?xml version='1.0' encoding='UTF-8'?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<container version='1.0'> + <search> + <chain id="default"> + <searcher id='com.yahoo.vespatest.ExtraHitSearcher' bundle='com.yahoo.vespatest.ExtraHitSearcher:1.2'> + <config name='vespatest.extra-hit'> + <exampleString>Heal the </exampleString> + <enumVal>World</enumVal> + </config> + </searcher> + + </chain> + </search> +</container> diff --git a/application/src/test/app-packages/withcontent/searchdefinitions/mydoc.sd b/application/src/test/app-packages/withcontent/searchdefinitions/mydoc.sd new file mode 100644 index 00000000000..4e557a0fc04 --- /dev/null +++ b/application/src/test/app-packages/withcontent/searchdefinitions/mydoc.sd @@ -0,0 +1,17 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search mydoc { + + document mydoc { + + field title type string { + indexing: index + } + + field substring type string { + indexing: index + match: gram + } + + } + +} diff --git a/application/src/test/app-packages/withcontent/services.xml b/application/src/test/app-packages/withcontent/services.xml new file mode 100644 index 00000000000..8f19a2f0bc8 --- /dev/null +++ b/application/src/test/app-packages/withcontent/services.xml @@ -0,0 +1,38 @@ +<?xml version='1.0' encoding='UTF-8'?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services> + + <jdisc version="1.0"> + + <search> + <chain id="default" inherits="vespa"/> + <provider id="bar" type="local" cluster="foo"> + <searcher id="MockResultSearcher" class="com.yahoo.application.MockResultSearcher"/> + </provider> + </search> + + <!-- TODO .. + <document-processing> + <chain id="default"> + <documentprocessor id="TestDocProc" class="com.yahoo.application.TestDocProc"> + <config name="foo.something"> + <variable>value</variable> + </config> + </documentprocessor> + </chain> + </document-processing> + --> + + </jdisc> + + <content version="1.0" id="foo"> + <redundancy>2</redundancy> + <documents> + <document type="mydoc" mode="index"/> + </documents> + <nodes> + <node hostalias="node1" distribution-key="1"/> + </nodes> + </content> + +</services> diff --git a/application/src/test/java/com/yahoo/application/ApplicationFacade.java b/application/src/test/java/com/yahoo/application/ApplicationFacade.java new file mode 100644 index 00000000000..2c8ff05e2cc --- /dev/null +++ b/application/src/test/java/com/yahoo/application/ApplicationFacade.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application; + +import com.google.common.annotations.Beta; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.component.Component; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.Container; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.DocumentOperation; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.service.ClientProvider; +import com.yahoo.jdisc.service.ServerProvider; +import com.yahoo.search.Query; +import com.yahoo.search.Result; + +/** + * Convenience access methods into some application content. + * These are not generally applicable and are therefore in test. + * This can also auto close its application. + * + * @author bratseth + */ +public class ApplicationFacade implements AutoCloseable { + + private static final String DEFAULT_CHAIN = "default"; + + private final Application application; + + public ApplicationFacade(Application application) { + this.application = application; + } + + /** Returns the application wrapped by this */ + public Application application() { return application; } + + /** + * Process a single document through the default chain. + * + * @param document the document to process + * @return the docproc return status + */ + @Beta + public DocumentProcessor.Progress process(final DocumentOperation document) { + return process(DEFAULT_CHAIN, document); + } + + /** + * Process a single document through the default chain. + * + * @param chain process using this chain + * @param op the document operation to process + * @return the docproc return status + */ + @Beta + public DocumentProcessor.Progress process(String chain, final DocumentOperation op) { + return application.getJDisc("default").documentProcessing().process(ComponentSpecification.fromString(chain), + Processing.of(op)); + } + + /** + * Pass a processing object through the + * + * @param processing the processing object to process + * @return the docproc return status + */ + @Beta + public DocumentProcessor.Progress process(final Processing processing) { + return process(DEFAULT_CHAIN, processing); + } + + /** + * Pass a processing object through the + * + * @param chain process using this chain + * @param processing the processing object to process + * @return the docproc return status + */ + @Beta + public DocumentProcessor.Progress process(String chain, final Processing processing) { + return application.getJDisc("default").documentProcessing().process(ComponentSpecification.fromString(chain), processing); + } + + /** + * Pass query object to the default search chain + * + * @param query the query + * @return the search result + */ + @Beta + public Result search(final Query query) { + return application.getJDisc("default").search().process(ComponentSpecification.fromString(DEFAULT_CHAIN), query); + } + + /** + * Pass query object to the default search chain + * + * @param chain the search chain to use + * @param query the query + * @return the search result + */ + @Beta + public Result search(String chain, final Query query) { + return application.getJDisc("default").search().process(ComponentSpecification.fromString(chain), query); + } + + /** + * Pass query object to the default search chain + * + * @param request the request to process + * @return the response object + */ + @Beta + public Response handleRequest(Request request) { + return application.getJDisc("default").handleRequest(request); + } + + /** + * @param componentId name of the component (usually YourClass.class.getName()) + * @return the component object, or null if not found + */ + @Beta + public Component getComponentById(String componentId) { + return Container.get().getComponentRegistry().getComponent(new ComponentId(componentId)); + } + + /** + * @param componentId name of the component (usually YourClass.class.getName()) + * @return the request handler object, or null if not found + */ + @Beta + public RequestHandler getRequestHandlerById(String componentId) { + return Container.get().getRequestHandlerRegistry().getComponent(new ComponentId(componentId)); + } + + /** + * @param componentId name of the component (usually YourClass.class.getName()) + * @return the client object, or null if not found + */ + @Beta + public ClientProvider getClientById(String componentId) { + return Container.get().getClientProviderRegistry().getComponent(new ComponentId(componentId)); + } + + /** + * @param componentId name of the component (usually YourClass.class.getName()) + * @return the client object, or null if not found + */ + @Beta + public ServerProvider getServerById(String componentId) { + return Container.get().getServerProviderRegistry().getComponent(new ComponentId(componentId)); + } + + @Override + public void close() throws Exception { + application.close(); + } +} diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java new file mode 100644 index 00000000000..7f14a3e4e16 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -0,0 +1,373 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application; + +import com.yahoo.application.container.MockClient; +import com.yahoo.application.container.MockServer; +import com.yahoo.application.container.docprocs.MockDispatchDocproc; +import com.yahoo.application.container.docprocs.MockDocproc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.application.container.handlers.MockHttpHandler; +import com.yahoo.application.container.renderers.MockRenderer; +import com.yahoo.application.container.searchers.MockSearcher; +import com.yahoo.component.Component; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.vespa.defaults.Defaults; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.ConnectException; +import java.net.ServerSocket; +import java.util.Map; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import static com.yahoo.application.container.JDiscTest.getListenPort; + +/** + * @author bratseth + */ +public class ApplicationTest { + + @Test + public void minimal_application_can_be_constructed() throws Exception { + try (Application application = Application.fromServicesXml("<jdisc version=\"1.0\"/>", Networking.disable)) { + } + } + + /** Tests that an application with search chains referencing a content cluster can be constructed. */ + @Test + public void container_and_referenced_content() throws Exception { + try (Application application = + Application.fromApplicationPackage(new File("src/test/app-packages/withcontent"), Networking.disable)) { + Result result = application.getJDisc("default").search().process(new ComponentSpecification("default"), + new Query("?query=substring:foobar&tracelevel=3")); + assertEquals("AND substring:fo substring:oo substring:ob substring:ba substring:ar", result.hits().get("hasQuery").getQuery().getModel().getQueryTree().toString()); + } + } + + private void printTrace(Result result) { + for (String message : result.getQuery().getContext(true).getTrace().traceNode().descendants(String.class)) + System.out.println(message); + } + + @Test + public void empty_container() throws Exception { + try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container())))) { + try { + app.process(new DocumentRemove(null)); + fail("expected exception"); + } catch (Exception ignore) { + // no op + } + + try { + app.process(new Processing()); + fail("expected exception"); + } catch (Exception ignore) { + // no op + } + + try { + app.search(new Query("?foo")); + fail("expected exception"); + } catch (Exception ignore) { + // no op + } + } + } + + @Test + public void config() throws Exception { + try ( + ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .documentProcessor("docproc", "default", MockDocproc.class) + .config(new MockApplicationConfig(new MockApplicationConfig.Builder() + .mystruct(new MockApplicationConfig.Mystruct.Builder().id("structid").value("structvalue")) + .mystructlist(new MockApplicationConfig.Mystructlist.Builder().id("listid1").value("listvalue1")) + .mystructlist(new MockApplicationConfig.Mystructlist.Builder().id("listid2").value("listvalue2")) + .mylist("item1") + .mylist("item2") + .mymap("key1", "value1") + .mymap("key2", "value2") + .mymapstruct("key1", new MockApplicationConfig.Mymapstruct.Builder().id("mapid1").value("mapvalue1")) + .mymapstruct("key2", new MockApplicationConfig.Mymapstruct.Builder().id("mapid2").value("mapvalue2"))))))) + ) { + + MockDocproc docproc = (MockDocproc) app.getComponentById("docproc@default"); + assertNotNull(docproc); + + // struct + assertEquals(docproc.getConfig().mystruct().id(), "structid"); + assertEquals(docproc.getConfig().mystruct().value(), "structvalue"); + + // struct list + assertEquals(docproc.getConfig().mystructlist().size(), 2); + assertEquals(docproc.getConfig().mystructlist().get(0).id(), "listid1"); + assertEquals(docproc.getConfig().mystructlist().get(0).value(), "listvalue1"); + assertEquals(docproc.getConfig().mystructlist().get(1).id(), "listid2"); + assertEquals(docproc.getConfig().mystructlist().get(1).value(), "listvalue2"); + + // list + assertEquals(docproc.getConfig().mylist().size(), 2); + assertEquals(docproc.getConfig().mylist().get(0), "item1"); + assertEquals(docproc.getConfig().mylist().get(1), "item2"); + + // map + assertEquals(docproc.getConfig().mymap().size(), 2); + assertTrue(docproc.getConfig().mymap().containsKey("key1")); + assertEquals(docproc.getConfig().mymap().get("key1"), "value1"); + assertTrue(docproc.getConfig().mymap().containsKey("key2")); + assertEquals(docproc.getConfig().mymap().get("key2"), "value2"); + + // map struct + assertEquals(docproc.getConfig().mymapstruct().size(), 2); + assertTrue(docproc.getConfig().mymapstruct().containsKey("key1")); + assertEquals(docproc.getConfig().mymapstruct().get("key1").id(), "mapid1"); + assertEquals(docproc.getConfig().mymapstruct().get("key1").value(), "mapvalue1"); + assertTrue(docproc.getConfig().mymapstruct().containsKey("key2")); + assertEquals(docproc.getConfig().mymapstruct().get("key2").id(), "mapid2"); + assertEquals(docproc.getConfig().mymapstruct().get("key2").value(), "mapvalue2"); + } + } + + @Test + public void handler() throws Exception { + try ( + ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .handler("http://*/*", MockHttpHandler.class)))) + ) { + + RequestHandler handler = app.getRequestHandlerById(MockHttpHandler.class.getName()); + assertNotNull(handler); + + Request request = new Request("http://localhost:" + Defaults.getDefaults().vespaWebServicePort() + "/"); + Response response = app.handleRequest(request); + assertNotNull(response); + assertEquals(response.getStatus(), 200); + assertEquals(response.getBodyAsString(), "OK"); + + request = new Request("http://localhost"); + response = app.handleRequest(request); + assertNotNull(response); + assertEquals(response.getStatus(), 200); + assertEquals(response.getBodyAsString(), "OK"); + + request = new Request("http://localhost/query=foo"); + response = app.handleRequest(request); + assertNotNull(response); + assertEquals(response.getStatus(), 200); + assertEquals(response.getBodyAsString(), "OK"); + } + } + + @Test + public void renderer() throws Exception { + try ( + ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .renderer("mock", MockRenderer.class)))) + ) { + + Request request = new Request("http://localhost:" + Defaults.getDefaults().vespaWebServicePort() + "/search/?format=mock"); + Response response = app.handleRequest(request); + assertNotNull(response); + assertEquals(response.getStatus(), 200); + assertEquals(response.getBodyAsString(), "<mock hits=\"0\" />"); + } + } + + @Test + public void search_default() throws Exception { + try ( + ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .searcher(MockSearcher.class)))) + ) { + Result result = app.search(new Query("?query=foo")); + assertEquals(1, result.hits().size()); + } + } + + @Test + public void search() throws Exception { + try ( + ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .searcher("foo", MockSearcher.class)))) + ) { + Result result = app.search("foo", new Query("?query=foo")); + assertEquals(1, result.hits().size()); + } + } + + @Test + public void builder_with_networking() throws Exception { + try ( + Application app = Application.fromBuilder(new Application.Builder().networking(Networking.enable).container("default", new Application.Builder.Container().handler("http://*/*", MockHttpHandler.class))) + ) { + DefaultHttpClient client = new DefaultHttpClient(); + HttpResponse response = client.execute(new HttpGet("http://localhost:" + getListenPort() + "/query=foo")); + assertEquals(200, response.getStatusLine().getStatusCode()); + BufferedReader r = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); + String line; + StringBuilder sb = new StringBuilder(); + while ((line = r.readLine()) != null) { + sb.append(line).append("\n"); + } + assertEquals("OK\n", sb.toString()); + } + } + + @Test + public void document_type() throws Exception { + try ( + Application app = Application.fromBuilder(new Application.Builder() + .documentType("test", IOUtils.toString(this.getClass().getResourceAsStream("/test.sd"))) + .container("default", new Application.Builder.Container() + .documentProcessor(MockDocproc.class) + .config(new MockApplicationConfig(new MockApplicationConfig.Builder().mystruct(new MockApplicationConfig.Mystruct.Builder().id("foo").value("bar")))))) + ) { + Map<String, DocumentType> typeMap = app.getJDisc("jdisc").documentProcessing().getDocumentTypes(); + assertNotNull(typeMap); + assertTrue(typeMap.containsKey("test")); + } + } + + @Test + public void get_search_handler() throws Exception { + try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container().search(true))))) { + SearchHandler searchHandler = (SearchHandler) app.getRequestHandlerById("com.yahoo.search.handler.SearchHandler"); + assertNotNull(searchHandler); + } + } + + @Test + public void component() throws Exception { + try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .component(MockSearcher.class))))) { + Component c = app.getComponentById(MockSearcher.class.getName()); + assertNotNull(c); + } + } + + @Test + public void component_with_config() throws Exception { + MockApplicationConfig config = new MockApplicationConfig(new MockApplicationConfig.Builder().mystruct(new MockApplicationConfig.Mystruct.Builder().id("foo").value("bar"))); + try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .component("foo", MockDocproc.class, config))))) { + Component c = app.getComponentById("foo"); + assertNotNull(c); + } + } + + @Test + public void client() throws Exception { + try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder() + .documentType("test", IOUtils.toString(this.getClass().getResourceAsStream("/test.sd"))) + .container("default", new Application.Builder.Container() + .client("mbus://*/*", MockClient.class) + .documentProcessor(MockDispatchDocproc.class) + )) + )) { + + Map<String, DocumentType> typeMap = app.application().getJDisc("jdisc").documentProcessing().getDocumentTypes(); + assertNotNull(typeMap); + + DocumentType docType = typeMap.get("test"); + Document doc = new Document(docType, "id:foo:test::bar"); + doc.setFieldValue("title", "hello"); + + assertEquals(DocumentProcessor.Progress.DONE, app.process(new DocumentPut(doc))); + + MockClient client = (MockClient) app.getClientById(MockClient.class.getName()); + assertNotNull(client); + assertEquals(1, client.getCounter()); + + MockDispatchDocproc docproc = (MockDispatchDocproc) app.getComponentById(MockDispatchDocproc.class.getName() + "@default"); + assertNotNull(docproc); + assertEquals(1, docproc.getResponses().size()); + assertEquals(200, docproc.getResponses().get(0).getStatus()); + } + } + + @Test + public void server() throws Exception { + try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() + .server("foo", MockServer.class))) + )) { + MockServer server = (MockServer) app.getServerById("foo"); + assertNotNull(server); + assertTrue(server.isStarted()); + } + } + + @Test + public void query_profile() throws Exception { + try (Application app = Application.fromBuilder(new Application.Builder() + .queryProfile("default", "<query-profile id=\"default\">\n" + + "<field name=\"defaultage\">7d</field>\n" + + "</query-profile>") + .queryProfileType("type", "<query-profile-type id=\"type\">\n" + + "<field name=\"defaultage\" type=\"string\" />\n" + + "</query-profile-type>") + .rankExpression("re", "commonfirstphase(globalstaticrank)") + .documentType("test", IOUtils.toString(this.getClass().getResourceAsStream("/test.sd"))) + .container("default", new Application.Builder.Container() + .search(true) + ))) { + } + } + + @Test(expected = ConnectException.class) + public void http_interface_is_off_when_networking_is_disabled() throws Exception { + int httpPort = getFreePort(); + try (Application application = Application.fromServicesXml(servicesXmlWithServer(httpPort), Networking.disable)) { + HttpClient client = new DefaultHttpClient(); + int statusCode = client.execute(new HttpGet("http://localhost:" + httpPort)).getStatusLine().getStatusCode(); + fail("Networking.disable is specified, but the network interface is enabled! Got status code: " + statusCode); + } + } + + @Test + public void http_interface_is_on_when_networking_is_enabled() throws Exception { + int httpPort = getFreePort(); + try (Application application = Application.fromServicesXml(servicesXmlWithServer(httpPort), Networking.enable)) { + HttpClient client = new DefaultHttpClient(); + int statusCode = client.execute(new HttpGet("http://localhost:" + httpPort)).getStatusLine().getStatusCode(); + assertEquals(200, statusCode); + } + } + + private static int getFreePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + socket.setReuseAddress(true); + return socket.getLocalPort(); + } + } + + private static String servicesXmlWithServer(int port) { + return "<jdisc version='1.0'>" + + " <http> <server port='" + port +"' id='foo'/> </http>" + + "</jdisc>"; + } + +} diff --git a/application/src/test/java/com/yahoo/application/MockResultSearcher.java b/application/src/test/java/com/yahoo/application/MockResultSearcher.java new file mode 100644 index 00000000000..2dd3a34a6ec --- /dev/null +++ b/application/src/test/java/com/yahoo/application/MockResultSearcher.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class MockResultSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = new Result(query); + result.hits().add(new Hit("hasQuery", query)); + return result; + } + +} diff --git a/application/src/test/java/com/yahoo/application/TestDocProc.java b/application/src/test/java/com/yahoo/application/TestDocProc.java new file mode 100644 index 00000000000..1631c7cd812 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/TestDocProc.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application; + +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.config.DocumentmanagerConfig; + +/** + * @author bratseth + */ +public class TestDocProc extends DocumentProcessor { + + public TestDocProc(DocumentmanagerConfig config) { + new ConfigSubscriber().subscribe(DocumentmanagerConfig.class, "client"); + } + + @Override + public Progress process(Processing processing) { + return Progress.DONE; + } + +} diff --git a/application/src/test/java/com/yahoo/application/container/JDiscContainerDocprocTest.java b/application/src/test/java/com/yahoo/application/container/JDiscContainerDocprocTest.java new file mode 100644 index 00000000000..d6c0e08a570 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/JDiscContainerDocprocTest.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.yahoo.application.Application; +import com.yahoo.application.ApplicationBuilder; +import com.yahoo.application.Networking; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.application.container.docprocs.Rot13DocumentProcessor; +import com.yahoo.container.Container; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.processing.execution.chain.ChainRegistry; +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class JDiscContainerDocprocTest { + + private static final String DOCUMENT = "document music {\n" + + " field title type string { }\n" + + "}\n"; + private static final String CHAIN_NAME = "myChain"; + + + private static String getXML(String chainName, String... processorIds) { + String xml = + "<container version=\"1.0\">\n" + + " <document-processing>\n" + + " <chain id=\"" + chainName + "\">\n"; + for (String processorId : processorIds) { + xml += " <documentprocessor id=\"" + processorId + "\"/>\n"; + } + xml += + " </chain>\n" + + " </document-processing>\n" + + "</container>\n"; + return xml; + } + + @Before + public void resetContainer() { + Container.resetInstance(); + } + + @Test + public void requireThatBasicDocumentProcessingWorks() throws Exception { + try (Application app = new ApplicationBuilder() + .servicesXml(getXML(CHAIN_NAME, Rot13DocumentProcessor.class.getCanonicalName())) + .documentType("music", DOCUMENT).build()) { + + JDisc container = app.getJDisc("container"); + DocumentProcessing docProc = container.documentProcessing(); + DocumentType type = docProc.getDocumentTypes().get("music"); + + ChainRegistry<DocumentProcessor> chains = docProc.getChains(); + assertTrue(chains.allComponentsById().containsKey(new ComponentId(CHAIN_NAME))); + + Document doc = new Document(type, "doc:this:is:a:great:album"); + doc.setFieldValue("title", "Great Album!"); + com.yahoo.docproc.Processing processing; + DocumentProcessor.Progress progress; + DocumentPut put = new DocumentPut(doc); + + processing = com.yahoo.docproc.Processing.of(put); + progress = docProc.process(ComponentSpecification.fromString(CHAIN_NAME), processing); + assertThat(progress, sameInstance(DocumentProcessor.Progress.DONE)); + assertThat(doc.getFieldValue("title").toString(), equalTo("Terng Nyohz!")); + + processing = com.yahoo.docproc.Processing.of(put); + progress = docProc.process(ComponentSpecification.fromString(CHAIN_NAME), processing); + assertThat(progress, sameInstance(DocumentProcessor.Progress.DONE)); + assertThat(doc.getFieldValue("title").toString(), equalTo("Great Album!")); + } + } + + @Test + public void requireThatLaterDocumentProcessingWorks() throws Exception { + try (Application app = new ApplicationBuilder() + .servicesXml(getXML(CHAIN_NAME, Rot13DocumentProcessor.class.getCanonicalName())) + .networking(Networking.disable) + .documentType("music", DOCUMENT).build()) { + JDisc container = app.getJDisc("container"); + DocumentProcessing docProc = container.documentProcessing(); + DocumentType type = docProc.getDocumentTypes().get("music"); + + ChainRegistry<DocumentProcessor> chains = docProc.getChains(); + assertTrue(chains.allComponentsById().containsKey(new ComponentId(CHAIN_NAME))); + + Document doc = new Document(type, "doc:this:is:a:great:album"); + doc.setFieldValue("title", "Great Album!"); + com.yahoo.docproc.Processing processing; + DocumentProcessor.Progress progress; + DocumentPut put = new DocumentPut(doc); + + processing = com.yahoo.docproc.Processing.of(put); + + progress = docProc.processOnce(ComponentSpecification.fromString(CHAIN_NAME), processing); + assertThat(progress, instanceOf(DocumentProcessor.LaterProgress.class)); + assertThat(doc.getFieldValue("title").toString(), equalTo("Great Album!")); + + progress = docProc.processOnce(ComponentSpecification.fromString(CHAIN_NAME), processing); + assertThat(progress, instanceOf(DocumentProcessor.LaterProgress.class)); + assertThat(doc.getFieldValue("title").toString(), equalTo("Great Album!")); + + progress = docProc.processOnce(ComponentSpecification.fromString(CHAIN_NAME), processing); + assertThat(progress, sameInstance(DocumentProcessor.Progress.DONE)); + assertThat(doc.getFieldValue("title").toString(), equalTo("Terng Nyohz!")); + } + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatUnknownChainThrows() { + try (JDisc container = JDisc.fromServicesXml( + getXML("foo", Rot13DocumentProcessor.class.getCanonicalName()), + Networking.disable)) { + container.documentProcessing().process(ComponentSpecification.fromString("unknown"), + new com.yahoo.docproc.Processing()); + } + + } + + + @Test(expected = UnsupportedOperationException.class) + public void requireThatProcessingFails() { + try (JDisc container = JDisc.fromServicesXml( + getXML("foo", Rot13DocumentProcessor.class.getCanonicalName()), + Networking.disable)) { + container.processing(); + } + + } + + @Test(expected = UnsupportedOperationException.class) + public void requireThatSearchFails() { + try (JDisc container = JDisc.fromServicesXml( + getXML("foo", Rot13DocumentProcessor.class.getCanonicalName()), + Networking.disable)) { + container.search(); + } + + } +} diff --git a/application/src/test/java/com/yahoo/application/container/JDiscContainerProcessingTest.java b/application/src/test/java/com/yahoo/application/container/JDiscContainerProcessingTest.java new file mode 100644 index 00000000000..a95068ce3a4 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/JDiscContainerProcessingTest.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.processors.Rot13Processor; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.Container; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class JDiscContainerProcessingTest { + + private static String getXML(String chainName, String... processorIds) { + String xml = + "<container version=\"1.0\">\n" + + " <processing>\n" + + " <chain id=\"" + chainName + "\">\n"; + for (String processorId : processorIds) { + xml += " <processor id=\"" + processorId + "\"/>\n"; + } + xml += + " </chain>\n" + + " </processing>\n" + + "</container>\n"; + return xml; + } + + @Before + public void resetContainer() { + Container.resetInstance(); + } + + @Test + public void requireThatBasicProcessingWorks() { + try (JDisc container = getContainerWithRot13()) { + Processing processing = container.processing(); + + Request req = new Request(); + req.properties().set("title", "Good day!"); + Response response = processing.process(ComponentSpecification.fromString("foo"), req); + + assertThat(response.data().get(0).toString(), equalTo("Tbbq qnl!")); + } + } + + @Test + public void requireThatBasicProcessingDoesNotTruncateBigResponse() { + final int SIZE = 50*1000; + StringBuilder foo = new StringBuilder(); + for (int j = 0 ; j < SIZE ; j++) { + foo.append('b'); + } + + try (JDisc container = getContainerWithRot13()) { + final int NUM_TIMES = 10000; + for (int i = 0; i < NUM_TIMES; i++) { + + + com.yahoo.application.container.handler.Response response = + container.handleRequest( + new com.yahoo.application.container.handler.Request("http://foo/processing/?chain=foo&title=" + foo.toString())); + + assertThat(response.getBody().length, is(SIZE+26)); + } + } + } + + @Test + public void processing_and_rendering_works() throws Exception { + try (JDisc container = getContainerWithRot13()) { + Processing processing = container.processing(); + + Request req = new Request(); + req.properties().set("title", "Good day!"); + + byte[] rendered = processing.processAndRender(ComponentSpecification.fromString("foo"), + ComponentSpecification.fromString("default"), req); + String renderedAsString = new String(rendered, "utf-8"); + + assertThat(renderedAsString, containsString("Tbbq qnl!")); + } + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatUnknownChainThrows() { + try (JDisc container = getContainerWithRot13()) { + container.processing().process(ComponentSpecification.fromString("unknown"), new Request()); + } + + } + + @Test(expected = UnsupportedOperationException.class) + public void requireThatDocprocFails() { + try (JDisc container = getContainerWithRot13()) { + container.documentProcessing(); + } + + } + + @Test(expected = UnsupportedOperationException.class) + public void requireThatSearchFails() { + try (JDisc container = getContainerWithRot13()) { + container.search(); + } + + } + + private JDisc getContainerWithRot13() { + return JDisc.fromServicesXml( + getXML("foo", Rot13Processor.class.getCanonicalName()), + Networking.disable); + } + +} diff --git a/application/src/test/java/com/yahoo/application/container/JDiscContainerRequestTest.java b/application/src/test/java/com/yahoo/application/container/JDiscContainerRequestTest.java new file mode 100644 index 00000000000..1ab91547f04 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/JDiscContainerRequestTest.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.application.container.handlers.DelayedThrowingInWriteRequestHandler; +import com.yahoo.application.container.handlers.DelayedWriteException; +import com.yahoo.application.container.handlers.EchoRequestHandler; +import com.yahoo.application.container.handlers.HeaderEchoRequestHandler; +import com.yahoo.application.container.handlers.ThrowingInWriteRequestHandler; +import com.yahoo.application.container.handlers.WriteException; +import com.yahoo.text.Utf8; +import org.junit.Test; + +import java.nio.charset.CharacterCodingException; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class JDiscContainerRequestTest { + + private static String getXML(String className, String binding) { + return "<container version=\"1.0\">\n" + + " <handler id=\"test-handler\" class=\"" + + className + + "\">\n" + + " <binding>" + + binding + + "</binding>\n" + + " </handler>\n" + + "</container>"; + } + + @Test + public void requireThatRequestBodyWorks() throws InterruptedException, CharacterCodingException { + final String DATA = "we have no bananas today"; + Request req = new Request("http://banana/echo", DATA.getBytes(Utf8.getCharset())); + + try (JDisc container = JDisc.fromServicesXml(getXML(EchoRequestHandler.class.getCanonicalName(), "http://*/echo"), Networking.disable)) { + Response response = container.handleRequest(req); + assertThat(response.getBodyAsString(), equalTo(DATA)); + req.toString(); + response.toString(); + } + } + + @Test + public void requireThatCustomRequestHeadersWork() throws InterruptedException { + Request req = new Request("http://banana/echo"); + req.getHeaders().add("X-Foo", "Bar"); + + try (JDisc container = JDisc.fromServicesXml(getXML(HeaderEchoRequestHandler.class.getCanonicalName(), "http://*/echo"), Networking.disable)) { + Response response = container.handleRequest(req); + assertThat(response.getHeaders().contains("X-Foo", "Bar"), is(true)); + req.toString(); + response.toString(); + } + } + + @Test(expected = WriteException.class) + public void requireThatRequestHandlerThatThrowsInWriteWorks() throws InterruptedException { + final String DATA = "we have no bananas today"; + Request req = new Request("http://banana/throwwrite", DATA.getBytes(Utf8.getCharset())); + + try (JDisc container = JDisc.fromServicesXml(getXML(ThrowingInWriteRequestHandler.class.getCanonicalName(), "http://*/throwwrite"), Networking.disable)) { + Response response = container.handleRequest(req); + req.toString(); + } + } + + + @Test(expected = DelayedWriteException.class) + public void requireThatRequestHandlerThatThrowsDelayedInWriteWorks() throws InterruptedException { + final String DATA = "we have no bananas today"; + Request req = new Request("http://banana/delayedthrowwrite", DATA.getBytes(Utf8.getCharset())); + + try (JDisc container = JDisc.fromServicesXml(getXML(DelayedThrowingInWriteRequestHandler.class.getCanonicalName(), "http://*/delayedthrowwrite"), Networking.disable)) { + Response response = container.handleRequest(req); + req.toString(); + } + } +} diff --git a/application/src/test/java/com/yahoo/application/container/MockClient.java b/application/src/test/java/com/yahoo/application/container/MockClient.java new file mode 100644 index 00000000000..1f885cfb737 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/MockClient.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container;
+
+import com.yahoo.jdisc.Request;
+import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.handler.CompletionHandler;
+import com.yahoo.jdisc.handler.ContentChannel;
+import com.yahoo.jdisc.handler.ResponseHandler;
+import com.yahoo.jdisc.service.AbstractClientProvider;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ *
+ * @author Christian Andersen
+ */
+public class MockClient extends AbstractClientProvider {
+ private final AtomicInteger counter = new AtomicInteger();
+
+ @Override
+ public ContentChannel handleRequest(Request request, ResponseHandler handler) {
+ counter.incrementAndGet();
+ final ContentChannel responseContentChannel = handler.handleResponse(new Response(200));
+ responseContentChannel.close(NOOP_COMPLETION_HANDLER);
+ return null;
+ }
+
+ public int getCounter() {
+ return counter.get();
+ }
+
+ private static final CompletionHandler NOOP_COMPLETION_HANDLER = new CompletionHandler() {
+ @Override
+ public void completed() {
+ // Ignored
+ }
+
+ @Override
+ public void failed(Throwable t) {
+ // Ignored
+ }
+ };
+}
diff --git a/application/src/test/java/com/yahoo/application/container/MockServer.java b/application/src/test/java/com/yahoo/application/container/MockServer.java new file mode 100644 index 00000000000..bc79671c13a --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/MockServer.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container;
+
+import com.yahoo.jdisc.service.AbstractServerProvider;
+import com.yahoo.jdisc.service.CurrentContainer;
+
+/**
+ *
+ * @author Christian Andersen
+ */
+public class MockServer extends AbstractServerProvider {
+ private boolean started = false;
+
+ public MockServer(CurrentContainer container) {
+ super(container);
+ }
+
+ @Override
+ public void start() {
+ started = true;
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ public boolean isStarted() {
+ return started;
+ }
+}
diff --git a/application/src/test/java/com/yahoo/application/container/docprocs/MockDispatchDocproc.java b/application/src/test/java/com/yahoo/application/container/docprocs/MockDispatchDocproc.java new file mode 100644 index 00000000000..3c11cbdad8a --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/docprocs/MockDispatchDocproc.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.docprocs;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.yahoo.docproc.DocumentProcessor;
+import com.yahoo.docproc.Processing;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.jdisc.Request;
+import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.handler.RequestDispatch;
+import com.yahoo.jdisc.service.CurrentContainer;
+import com.yahoo.messagebus.jdisc.MbusRequest;
+import com.yahoo.messagebus.routing.Route;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * @author Christian Andersen
+ */
+public class MockDispatchDocproc extends DocumentProcessor {
+ private final Route route;
+ private final URI uri;
+ private final CurrentContainer currentContainer;
+ private final List<Response> responses = new ArrayList<>();
+
+ public MockDispatchDocproc(CurrentContainer currentContainer) {
+ this.route = Route.parse("default");
+ this.uri = URI.create("mbus://remotehost/source");
+ this.currentContainer = currentContainer;
+ }
+
+ @Override
+ public Progress process(Processing processing) {
+ for (DocumentOperation op : processing.getDocumentOperations()) {
+ PutDocumentMessage message = new PutDocumentMessage((DocumentPut)op);
+ ListenableFuture<Response> future = createRequest(message).dispatch();
+ try {
+ responses.add(future.get());
+ } catch (ExecutionException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return Progress.DONE;
+ }
+
+ private RequestDispatch createRequest(final DocumentMessage message) {
+ return new RequestDispatch() {
+ @Override
+ protected Request newRequest() {
+ return new MbusRequest(currentContainer, uri, message.setRoute(route)).setServerRequest(false);
+ }
+ };
+ }
+
+ public List<Response> getResponses() {
+ return responses;
+ }
+}
diff --git a/application/src/test/java/com/yahoo/application/container/docprocs/MockDocproc.java b/application/src/test/java/com/yahoo/application/container/docprocs/MockDocproc.java new file mode 100644 index 00000000000..c07655cfe5c --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/docprocs/MockDocproc.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.docprocs; + +import com.google.inject.Inject; +import com.yahoo.application.MockApplicationConfig; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; + +/** + * @author Christian Andersen + */ +public class MockDocproc extends DocumentProcessor { + private Processing lastProcessing; + private MockApplicationConfig config; + + @Inject + public MockDocproc(MockApplicationConfig config) { + this.config = config; + } + + @Override + public Progress process(Processing processing) { + this.lastProcessing = processing; + return Progress.DONE; + } + + public Processing getLastProcessing() { + return lastProcessing; + } + + public MockApplicationConfig getConfig() { + return config; + } +} diff --git a/application/src/test/java/com/yahoo/application/container/docprocs/Rot13DocumentProcessor.java b/application/src/test/java/com/yahoo/application/container/docprocs/Rot13DocumentProcessor.java new file mode 100644 index 00000000000..ec23b3671b9 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/docprocs/Rot13DocumentProcessor.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.docprocs; + +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.datatypes.StringFieldValue; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class Rot13DocumentProcessor extends DocumentProcessor { + private static final String FIELD_NAME = "title"; + + private AtomicInteger counter = new AtomicInteger(1); + + public static String rot13(String s) { + StringBuilder output = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c >= 'a' && c <= 'm' || c >= 'A' && c <= 'M') { + c += 13; + } else if (c >= 'n' && c <= 'z' || c >= 'N' && c <= 'Z') { + c -= 13; + } + output.append(c); + } + return output.toString(); + } + + @Override + public Progress process(Processing processing) { + int oldVal = counter.getAndIncrement(); + if ((oldVal % 3) != 0) { + return Progress.LATER; + } + + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op instanceof DocumentPut) { + Document document = ((DocumentPut)op).getDocument(); + + StringFieldValue oldTitle = (StringFieldValue) document.getFieldValue(FIELD_NAME); + if (oldTitle != null) { + document.setFieldValue(FIELD_NAME, rot13(oldTitle.getString())); + } + } + } + return Progress.DONE; + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handler/HeadersTestCase.java b/application/src/test/java/com/yahoo/application/container/handler/HeadersTestCase.java new file mode 100644 index 00000000000..7f4a32c7df6 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handler/HeadersTestCase.java @@ -0,0 +1,366 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handler; + +import org.junit.Test; + +import java.util.*; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertNotNull; +import static junit.framework.Assert.assertNull; +import static junit.framework.Assert.assertTrue; +import static junit.framework.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class HeadersTestCase { + + @Test + public void requireThatSizeWorksAsExpected() { + Headers headers = new Headers(); + assertEquals(0, headers.size()); + headers.add("foo", "bar"); + assertEquals(1, headers.size()); + headers.add("foo", "baz"); + assertEquals(1, headers.size()); + headers.add("bar", "baz"); + assertEquals(2, headers.size()); + headers.remove("foo"); + assertEquals(1, headers.size()); + headers.remove("bar"); + assertEquals(0, headers.size()); + } + + @Test + public void requireThatIsEmptyWorksAsExpected() { + Headers headers = new Headers(); + assertTrue(headers.isEmpty()); + headers.add("foo", "bar"); + assertFalse(headers.isEmpty()); + headers.remove("foo"); + assertTrue(headers.isEmpty()); + } + + @Test + public void requireThatContainsKeyWorksAsExpected() { + Headers headers = new Headers(); + assertFalse(headers.containsKey("foo")); + assertFalse(headers.containsKey("FOO")); + headers.add("foo", "bar"); + assertTrue(headers.containsKey("foo")); + assertTrue(headers.containsKey("FOO")); + } + + @Test + public void requireThatContainsValueWorksAsExpected() { + Headers headers = new Headers(); + assertFalse(headers.containsValue(Arrays.asList("bar"))); + headers.add("foo", "bar"); + assertTrue(headers.containsValue(Arrays.asList("bar"))); + } + + @Test + public void requireThatContainsWorksAsExpected() { + Headers headers = new Headers(); + assertFalse(headers.contains("foo", "bar")); + assertFalse(headers.contains("FOO", "bar")); + assertFalse(headers.contains("foo", "BAR")); + assertFalse(headers.contains("FOO", "BAR")); + headers.add("foo", "bar"); + assertTrue(headers.contains("foo", "bar")); + assertTrue(headers.contains("FOO", "bar")); + assertFalse(headers.contains("foo", "BAR")); + assertFalse(headers.contains("FOO", "BAR")); + } + + @Test + public void requireThatContainsIgnoreCaseWorksAsExpected() { + Headers headers = new Headers(); + assertFalse(headers.containsIgnoreCase("foo", "bar")); + assertFalse(headers.containsIgnoreCase("FOO", "bar")); + assertFalse(headers.containsIgnoreCase("foo", "BAR")); + assertFalse(headers.containsIgnoreCase("FOO", "BAR")); + headers.add("foo", "bar"); + assertTrue(headers.containsIgnoreCase("foo", "bar")); + assertTrue(headers.containsIgnoreCase("FOO", "bar")); + assertTrue(headers.containsIgnoreCase("foo", "BAR")); + assertTrue(headers.containsIgnoreCase("FOO", "BAR")); + } + + @Test + public void requireThatAddStringWorksAsExpected() { + Headers headers = new Headers(); + assertNull(headers.get("foo")); + headers.add("foo", "bar"); + assertEquals(Arrays.asList("bar"), headers.get("foo")); + headers.add("foo", "baz"); + assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); + } + + @Test + public void requireThatAddListWorksAsExpected() { + Headers headers = new Headers(); + assertNull(headers.get("foo")); + headers.add("foo", Arrays.asList("bar")); + assertEquals(Arrays.asList("bar"), headers.get("foo")); + headers.add("foo", Arrays.asList("baz", "cox")); + assertEquals(Arrays.asList("bar", "baz", "cox"), headers.get("foo")); + } + + @Test + public void requireThatAddAllWorksAsExpected() { + Headers headers = new Headers(); + headers.add("foo", "bar"); + headers.add("bar", "baz"); + assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(Arrays.asList("baz"), headers.get("bar")); + + Map<String, List<String>> map = new HashMap<>(); + map.put("foo", Arrays.asList("baz", "cox")); + map.put("bar", Arrays.asList("cox")); + headers.addAll(map); + + assertEquals(Arrays.asList("bar", "baz", "cox"), headers.get("foo")); + assertEquals(Arrays.asList("baz", "cox"), headers.get("bar")); + } + + @Test + public void requireThatPutStringWorksAsExpected() { + Headers headers = new Headers(); + assertNull(headers.get("foo")); + headers.put("foo", "bar"); + assertEquals(Arrays.asList("bar"), headers.get("foo")); + headers.put("foo", "baz"); + assertEquals(Arrays.asList("baz"), headers.get("foo")); + } + + @Test + public void requireThatPutListWorksAsExpected() { + Headers headers = new Headers(); + assertNull(headers.get("foo")); + headers.put("foo", Arrays.asList("bar")); + assertEquals(Arrays.asList("bar"), headers.get("foo")); + headers.put("foo", Arrays.asList("baz", "cox")); + assertEquals(Arrays.asList("baz", "cox"), headers.get("foo")); + } + + @Test + public void requireThatPutAllWorksAsExpected() { + Headers headers = new Headers(); + headers.add("foo", "bar"); + headers.add("bar", "baz"); + assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(Arrays.asList("baz"), headers.get("bar")); + + Map<String, List<String>> map = new HashMap<>(); + map.put("foo", Arrays.asList("baz", "cox")); + map.put("bar", Arrays.asList("cox")); + headers.putAll(map); + + assertEquals(Arrays.asList("baz", "cox"), headers.get("foo")); + assertEquals(Arrays.asList("cox"), headers.get("bar")); + } + + @Test + public void requireThatRemoveWorksAsExpected() { + Headers headers = new Headers(); + headers.put("foo", Arrays.asList("bar", "baz")); + assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); + assertEquals(Arrays.asList("bar", "baz"), headers.remove("foo")); + assertNull(headers.get("foo")); + assertNull(headers.remove("foo")); + } + + @Test + public void requireThatRemoveStringWorksAsExpected() { + Headers headers = new Headers(); + headers.put("foo", Arrays.asList("bar", "baz")); + assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); + assertTrue(headers.remove("foo", "bar")); + assertFalse(headers.remove("foo", "cox")); + assertEquals(Arrays.asList("baz"), headers.get("foo")); + assertTrue(headers.remove("foo", "baz")); + assertFalse(headers.remove("foo", "cox")); + assertNull(headers.get("foo")); + } + + @Test + public void requireThatClearWorksAsExpected() { + Headers headers = new Headers(); + headers.add("foo", "bar"); + headers.add("bar", "baz"); + assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(Arrays.asList("baz"), headers.get("bar")); + headers.clear(); + assertNull(headers.get("foo")); + assertNull(headers.get("bar")); + } + + @Test + public void requireThatGetWorksAsExpected() { + Headers headers = new Headers(); + assertNull(headers.get("foo")); + headers.add("foo", "bar"); + assertEquals(Arrays.asList("bar"), headers.get("foo")); + } + + @Test + public void requireThatGetFirstWorksAsExpected() { + Headers headers = new Headers(); + assertNull(headers.getFirst("foo")); + headers.add("foo", Arrays.asList("bar", "baz")); + assertEquals("bar", headers.getFirst("foo")); + } + + @Test + public void requireThatIsTrueWorksAsExpected() { + Headers headers = new Headers(); + assertFalse(headers.isTrue("foo")); + headers.put("foo", Arrays.asList("true")); + assertTrue(headers.isTrue("foo")); + headers.put("foo", Arrays.asList("true", "true")); + assertTrue(headers.isTrue("foo")); + headers.put("foo", Arrays.asList("true", "false")); + assertFalse(headers.isTrue("foo")); + headers.put("foo", Arrays.asList("false", "true")); + assertFalse(headers.isTrue("foo")); + headers.put("foo", Arrays.asList("false", "false")); + assertFalse(headers.isTrue("foo")); + headers.put("foo", Arrays.asList("false")); + assertFalse(headers.isTrue("foo")); + } + + @Test + public void requireThatKeySetWorksAsExpected() { + Headers headers = new Headers(); + assertEquals(Collections.<Set<String>>emptySet(), headers.keySet()); + headers.add("foo", "bar"); + assertEquals(new HashSet<>(Arrays.asList("foo")), headers.keySet()); + headers.add("bar", "baz"); + assertEquals(new HashSet<>(Arrays.asList("foo", "bar")), headers.keySet()); + } + + @Test + public void requireThatValuesWorksAsExpected() { + Headers headers = new Headers(); + assertTrue(headers.values().isEmpty()); + headers.add("foo", "bar"); + Collection<List<String>> values = headers.values(); + assertEquals(1, values.size()); + assertTrue(values.contains(Arrays.asList("bar"))); + + headers.add("bar", "baz"); + values = headers.values(); + assertEquals(2, values.size()); + assertTrue(values.contains(Arrays.asList("bar"))); + assertTrue(values.contains(Arrays.asList("baz"))); + } + + @Test + public void requireThatEntrySetWorksAsExpected() { + Headers headers = new Headers(); + assertEquals(Collections.emptySet(), headers.entrySet()); + headers.put("foo", Arrays.asList("bar", "baz")); + + Set<Map.Entry<String, List<String>>> entries = headers.entrySet(); + assertEquals(1, entries.size()); + Map.Entry<String, List<String>> entry = entries.iterator().next(); + assertNotNull(entry); + assertEquals("foo", entry.getKey()); + assertEquals(Arrays.asList("bar", "baz"), entry.getValue()); + } + + @Test + public void requireThatEntriesWorksAsExpected() { + Headers headers = new Headers(); + assertEquals(Collections.emptyList(), headers.entries()); + headers.put("foo", Arrays.asList("bar", "baz")); + + List<Map.Entry<String, String>> entries = headers.entries(); + assertEquals(2, entries.size()); + + Map.Entry<String, String> entry = entries.get(0); + assertNotNull(entry); + assertEquals("foo", entry.getKey()); + assertEquals("bar", entry.getValue()); + + assertNotNull(entry = entries.get(1)); + assertEquals("foo", entry.getKey()); + assertEquals("baz", entry.getValue()); + } + + @Test + public void requireThatEntryIsUnmodifiable() { + Headers headers = new Headers(); + headers.put("foo", "bar"); + Map.Entry<String, String> entry = headers.entries().get(0); + try { + entry.setValue("baz"); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + @Test + public void requireThatEntriesAreUnmodifiable() { + Headers headers = new Headers(); + headers.put("foo", "bar"); + List<Map.Entry<String, String>> entries = headers.entries(); + try { + entries.add(new MyEntry()); + fail(); + } catch (UnsupportedOperationException e) { + + } + try { + entries.remove(new MyEntry()); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + @Test + public void requireThatEqualsWorksAsExpected() { + Headers lhs = new Headers(); + Headers rhs = new Headers(); + assertTrue(lhs.equals(rhs)); + lhs.add("foo", "bar"); + assertFalse(lhs.equals(rhs)); + rhs.add("foo", "bar"); + assertTrue(lhs.equals(rhs)); + } + + @Test + public void requireThatHashCodeWorksAsExpected() { + Headers lhs = new Headers(); + Headers rhs = new Headers(); + assertTrue(lhs.hashCode() == rhs.hashCode()); + lhs.add("foo", "bar"); + assertTrue(lhs.hashCode() != rhs.hashCode()); + rhs.add("foo", "bar"); + assertTrue(lhs.hashCode() == rhs.hashCode()); + } + + private static class MyEntry implements Map.Entry<String, String> { + + @Override + public String getKey() { + return "key"; + } + + @Override + public String getValue() { + return "value"; + } + + @Override + public String setValue(String value) { + return "value"; + } + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handler/ResponseTestCase.java b/application/src/test/java/com/yahoo/application/container/handler/ResponseTestCase.java new file mode 100644 index 00000000000..53b62753523 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handler/ResponseTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handler; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ResponseTestCase { + + @Test + public void requireThatCharsetParsingWorks() { + assertThat(Response.charset("text/foobar").toString().toLowerCase(), equalTo("utf-8")); + assertThat(Response.charset("adsf").toString().toLowerCase(), equalTo("utf-8")); + assertThat(Response.charset("").toString().toLowerCase(), equalTo("utf-8")); + assertThat(Response.charset(null).toString().toLowerCase(), equalTo("utf-8")); + + assertThat(Response.charset("something; charset=US-ASCII").toString().toLowerCase(), equalTo("us-ascii")); + assertThat(Response.charset("something; charset=iso-8859-1").toString().toLowerCase(), equalTo("iso-8859-1")); + + assertThat(Response.charset("something; charset=").toString().toLowerCase(), equalTo("utf-8")); + assertThat(Response.charset("something; charset=bananarama").toString().toLowerCase(), equalTo("utf-8")); + } + + @Test + public void testDefaultResponseBody() { + Response res1 = new Response(); + Response res2 = new Response(new byte[0]); + + assertThat(res1.getBody(), notNullValue()); + assertThat(res1.getBody().length, is(0)); + assertThat(res2.getBody(), notNullValue()); + assertThat(res2.getBody().length, is(0)); + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handlers/DelayedThrowingInWriteRequestHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/DelayedThrowingInWriteRequestHandler.java new file mode 100644 index 00000000000..54c850efd37 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handlers/DelayedThrowingInWriteRequestHandler.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers; + +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** +* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> +*/ +public class DelayedThrowingInWriteRequestHandler extends AbstractRequestHandler { + private ExecutorService responseExecutor = Executors.newSingleThreadExecutor(); + + @Override + public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { + responseExecutor.execute(new DelayedThrowingInWriteTask(handler)); + return new DelayedThrowingInWriteContentChannel(); + } + + + private static class DelayedThrowingInWriteTask implements Runnable { + private final ResponseHandler handler; + + public DelayedThrowingInWriteTask(ResponseHandler handler) { + this.handler = handler; + } + + @Override + public void run() { + ContentChannel responseChannel = handler.handleResponse( + new com.yahoo.jdisc.Response(com.yahoo.jdisc.Response.Status.OK)); + responseChannel.close(null); + } + } + + + private static class DelayedThrowingInWriteContentChannel implements ContentChannel { + private List<CompletionHandler> writeCompletionHandlers = new ArrayList<>(); + @Override + public void write(ByteBuffer buf, CompletionHandler handler) { + writeCompletionHandlers.add(handler); + } + + @Override + public void close(CompletionHandler handler) { + for (CompletionHandler writeCompletionHandler : writeCompletionHandlers) { + writeCompletionHandler.failed(new DelayedWriteException()); + } + handler.completed(); + } + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handlers/DelayedWriteException.java b/application/src/test/java/com/yahoo/application/container/handlers/DelayedWriteException.java new file mode 100644 index 00000000000..e856ac310cb --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handlers/DelayedWriteException.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DelayedWriteException extends RuntimeException { +} diff --git a/application/src/test/java/com/yahoo/application/container/handlers/EchoRequestHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/EchoRequestHandler.java new file mode 100644 index 00000000000..30fe379d864 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handlers/EchoRequestHandler.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers; + +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; + +/** +* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> +*/ +public class EchoRequestHandler extends AbstractRequestHandler { + @Override + public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { + return handler.handleResponse(new com.yahoo.jdisc.Response(Response.Status.OK)); + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handlers/HeaderEchoRequestHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/HeaderEchoRequestHandler.java new file mode 100644 index 00000000000..4eea69ea162 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handlers/HeaderEchoRequestHandler.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers; + +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; + +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** +* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> +*/ +public class HeaderEchoRequestHandler extends AbstractRequestHandler { + private ExecutorService responseExecutor = Executors.newSingleThreadExecutor(); + + @Override + public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { + responseExecutor.execute(new HeaderEchoTask(request, handler)); + return new HeaderEchoContentChannel(); + } + + + private static class HeaderEchoTask implements Runnable { + private final Request request; + private final ResponseHandler handler; + + public HeaderEchoTask(Request request, ResponseHandler handler) { + this.request = request; + this.handler = handler; + } + + @Override + public void run() { + com.yahoo.jdisc.Response response = new com.yahoo.jdisc.Response(com.yahoo.jdisc.Response.Status.OK); + response.headers().addAll(request.headers()); + ContentChannel responseChannel = handler.handleResponse(response); + responseChannel.close(null); + } + } + + + private static class HeaderEchoContentChannel implements ContentChannel { + @Override + public void write(ByteBuffer buf, CompletionHandler handler) { + //we will not accept header body data + handler.completed(); + } + + @Override + public void close(CompletionHandler handler) { + //we will not accept header body data + handler.completed(); + } + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handlers/MockHttpHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/MockHttpHandler.java new file mode 100644 index 00000000000..b3287015b28 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handlers/MockHttpHandler.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.concurrent.Executor; + +/** + * + * @author Christian Andersen + */ +public class MockHttpHandler extends ThreadedHttpRequestHandler { + public MockHttpHandler(Executor executor) { + super(executor); + } + + @Override + public HttpResponse handle(HttpRequest request) { + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + PrintStream out = new PrintStream(outputStream); + out.print("OK"); + out.flush(); + } + }; + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handlers/ThrowingInWriteRequestHandler.java b/application/src/test/java/com/yahoo/application/container/handlers/ThrowingInWriteRequestHandler.java new file mode 100644 index 00000000000..9cde4eaf9f8 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handlers/ThrowingInWriteRequestHandler.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers; + +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; + +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** +* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> +*/ +public class ThrowingInWriteRequestHandler extends AbstractRequestHandler { + private ExecutorService responseExecutor = Executors.newSingleThreadExecutor(); + + @Override + public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { + responseExecutor.execute(new ThrowingInWriteTask(handler)); + return new ThrowingInWriteContentChannel(); + } + + + private static class ThrowingInWriteTask implements Runnable { + private final ResponseHandler handler; + + public ThrowingInWriteTask(ResponseHandler handler) { + this.handler = handler; + } + + @Override + public void run() { + ContentChannel responseChannel = handler.handleResponse( + new com.yahoo.jdisc.Response(com.yahoo.jdisc.Response.Status.OK)); + responseChannel.close(null); + } + } + + + private static class ThrowingInWriteContentChannel implements ContentChannel { + @Override + public void write(ByteBuffer buf, CompletionHandler handler) { + throw new WriteException(); + } + + @Override + public void close(CompletionHandler handler) { + handler.completed(); + } + } +} diff --git a/application/src/test/java/com/yahoo/application/container/handlers/WriteException.java b/application/src/test/java/com/yahoo/application/container/handlers/WriteException.java new file mode 100644 index 00000000000..63a62340efe --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/handlers/WriteException.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class WriteException extends RuntimeException { +} diff --git a/application/src/test/java/com/yahoo/application/container/processors/Rot13Processor.java b/application/src/test/java/com/yahoo/application/container/processors/Rot13Processor.java new file mode 100644 index 00000000000..f0ec8c25a88 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/processors/Rot13Processor.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.processors; + +import com.yahoo.processing.Processor; +import com.yahoo.processing.Request; +import com.yahoo.processing.Response; +import com.yahoo.processing.execution.Execution; +import com.yahoo.processing.test.ProcessorLibrary; + +import static com.yahoo.application.container.docprocs.Rot13DocumentProcessor.rot13; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class Rot13Processor extends Processor { + @Override + public Response process(Request request, Execution execution) { + Object fooObj = request.properties().get("title"); + + Response response = new Response(request); + if (fooObj != null) { + response.data().add(new ProcessorLibrary.StringData(request, rot13(fooObj.toString()))); + } + return response; + } +} diff --git a/application/src/test/java/com/yahoo/application/container/renderers/MockRenderer.java b/application/src/test/java/com/yahoo/application/container/renderers/MockRenderer.java new file mode 100644 index 00000000000..6e9899c7b9f --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/renderers/MockRenderer.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.renderers; + +import com.yahoo.search.Result; +import com.yahoo.search.rendering.Renderer; + +import java.io.IOException; +import java.io.Writer; + +/** + * + * @author Christian Andersen + */ +public class MockRenderer extends Renderer { + public MockRenderer() { + } + + @Override + public String getEncoding() { + return "utf-8"; + } + + @Override + public String getMimeType() { + return "applications/xml"; + } + + @Override + protected void render(Writer writer, Result result) throws IOException { + writer.write("<mock hits=\"" + result.hits().size() + "\" />"); + } +} diff --git a/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java b/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java new file mode 100644 index 00000000000..ad65e597519 --- /dev/null +++ b/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.searchers;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ *
+ * @author Christian Andersen
+ */
+public class MockSearcher extends Searcher {
+ @Override
+ public Result search(Query query, Execution execution) {
+ HitGroup hits = new HitGroup();
+ hits.add(new Hit("foo", query));
+ return new Result(query, hits);
+ }
+}
diff --git a/application/src/test/resources/configdefinitions/mock-application.def b/application/src/test/resources/configdefinitions/mock-application.def new file mode 100644 index 00000000000..bc61f976c15 --- /dev/null +++ b/application/src/test/resources/configdefinitions/mock-application.def @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +namespace=application + +mystruct.id string +mystruct.value string +mystructlist[].id string +mystructlist[].value string +mylist[] string +mymap{} string +mymapstruct{}.id string +mymapstruct{}.value string diff --git a/application/src/test/resources/test.sd b/application/src/test/resources/test.sd new file mode 100644 index 00000000000..ff196e6b189 --- /dev/null +++ b/application/src/test/resources/test.sd @@ -0,0 +1,23 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search test { + + document test { + + field title type string { + # index-to: title, default + weight: 30 + stemming: none + bolding: on + body + } + + field body type string { + # index-to: body, default + weight: 6 + stemming: none + summary: dynamic + body + } + } + +} diff --git a/application/src/test/scala/com/yahoo/application/ApplicationBuilderTest.scala b/application/src/test/scala/com/yahoo/application/ApplicationBuilderTest.scala new file mode 100644 index 00000000000..b73f20fc640 --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/ApplicationBuilderTest.scala @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application + +import container.JDiscTest._ +import java.nio.file.Files +import org.junit.Assert.{assertTrue, assertThat, fail} +import org.junit.Test +import com.yahoo.io.IOUtils +import org.junit.matchers.JUnitMatchers.containsString + +/** + * @author tonytv + */ + class ApplicationBuilderTest { + @Test + def query_profile_types_can_be_added() { + withApplicationBuilder { builder => + builder.queryProfileType("MyProfileType", + <query-profile-type id="MyProfileType"> + <field name="age" type="integer" /> + <field name="profession" type="string" /> + <field name="user" type="query-profile:MyUserProfile" /> + </query-profile-type>) + + assertTrue(Files.exists(builder.getPath.resolve("search/query-profiles/types/MyProfileType.xml"))) + } + } + + + @Test + def query_profile_can_be_added() { + withApplicationBuilder { builder => + builder.queryProfile("MyProfile", + <query-profile id="MyProfile"> + <field name="message">Hello world!</field> + </query-profile>) + + assertTrue(Files.exists(builder.getPath.resolve("search/query-profiles/MyProfile.xml"))) + } + } + + @Test + def rank_expression_can_be_added() { + withApplicationBuilder { builder => + builder.rankExpression("myExpression", "content") + assertTrue(Files.exists(builder.getPath.resolve("searchdefinitions/myExpression.expression"))) + } + } + + @Test + def builder_cannot_be_reused() { + val builder = new ApplicationBuilder + builder.servicesXml(<jdisc version="1.0" />) + + using(builder.build()) { builder => } + + try { + builder.servicesXml("") + fail("Expected exception.") + } catch { + case e: RuntimeException => assertThat(e.getMessage, containsString("build method")) + } + + } + + def withApplicationBuilder(f: ApplicationBuilder => Unit) { + val builder = new ApplicationBuilder() + try { + f(builder) + } finally { + IOUtils.recursiveDeleteDir(builder.getPath.toFile) + } + } +} diff --git a/application/src/test/scala/com/yahoo/application/container/JDiscContainerSearchTest.scala b/application/src/test/scala/com/yahoo/application/container/JDiscContainerSearchTest.scala new file mode 100644 index 00000000000..03b7cc566fe --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/container/JDiscContainerSearchTest.scala @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container + +import org.junit.Test +import searchers.AddHitSearcher +import com.yahoo.component.ComponentSpecification +import com.yahoo.search.Query +import org.junit.Assert._ +import org.hamcrest.CoreMatchers._ +import org.hamcrest.Matchers.containsString; +import JDiscTest.{fromServicesXml, using} + + +/** + * + * @author gjoranv + * @since 5.1.15 + */ +class JDiscContainerSearchTest { + + @Test + def processing_and_rendering_works() { + val searcherId = classOf[AddHitSearcher].getName + + using(containerWithSearch(searcherId)) + { container => + val rendered = container.search.processAndRender(ComponentSpecification.fromString("mychain"), + ComponentSpecification.fromString("DefaultRenderer"), new Query("")) + val renderedAsString = new String(rendered, "utf-8") + assertThat(renderedAsString, containsString(searcherId)) + } + } + + @Test + def searching_works() { + val searcherId = classOf[AddHitSearcher].getName + + using(containerWithSearch(searcherId)) + { container => + val searching = container.search + val result = searching.process(ComponentSpecification.fromString("mychain"), new Query("")) + + val hitTitle = result.hits().get(0).getField("title").asInstanceOf[String] + assertThat(hitTitle, is(searcherId)) + } + } + + def containerWithSearch(searcherId: String) = { + fromServicesXml( + <container version="1.0"> + <search> + <chain id="mychain"> + <searcher id={searcherId}/> + </chain> + </search> + </container>) + } + + @Test(expected = classOf[UnsupportedOperationException]) + def retrieving_search_from_container_without_search_is_illegal() { + using(JDiscTest.fromServicesXml( + <container version="1.0" /> + )) + { container => + container.search // throws + } + } + +} diff --git a/application/src/test/scala/com/yahoo/application/container/JDiscTest.scala b/application/src/test/scala/com/yahoo/application/container/JDiscTest.scala new file mode 100644 index 00000000000..aef1d540ee3 --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/container/JDiscTest.scala @@ -0,0 +1,199 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container + +import com.yahoo.container.Container +import com.yahoo.jdisc.http.server.jetty.JettyHttpServer + +import scala.language.implicitConversions +import handler.Request +import org.junit.{Ignore, Test} +import org.junit.Assert.{assertThat, assertNotNull, fail} +import org.hamcrest.CoreMatchers.is +import java.nio.file.FileSystems +import com.yahoo.search.Query +import com.yahoo.component.ComponentSpecification +import handlers.TestHandler +import xml.{Node, Elem} +import JDiscTest._ +import org.junit.matchers.JUnitMatchers.{containsString, hasItem} +import com.yahoo.application.{Networking, ApplicationBuilder} + +import scala.collection.convert.wrapAsScala._ + + +/** + * @author tonytv + * @author gjoranv + */ +class JDiscTest { + @Test + def jdisc_can_be_used_as_top_level_element() { + using(fromServicesXml( + <jdisc version="1.0"> + <search /> + </jdisc>)) + { container => + assertNotNull(container.search()) + } + } + + @Test + def jdisc_id_can_be_set() { + using(fromServicesXml( + <jdisc version="1.0" id="my-service-id"> + <search /> + </jdisc>)) + { container => + assertNotNull(container.search()) + } + } + + @Test + def jdisc_can_be_embedded_in_services_tag() { + using(fromServicesXml( + <services> + <jdisc version="1.0" id="my-service-id"> + <search /> + </jdisc> + </services>)) + { container => + assertNotNull(container.search()) + } + } + + @Test + def multiple_jdisc_elements_gives_exception() { + try { + using(fromServicesXml( + <services> + <jdisc version="1.0" id="id1" /> + <jdisc version="1.0" /> + <container version="1.0"/> + </services>)) + { container => fail("expected exception")} + } catch { + case e: Exception => assertThat(e.getMessage, containsString("container id='', jdisc id='id1', jdisc id=''")) + } + } + + @Test + def handleRequest_yields_response_from_correct_request_handler() { + using(fromServicesXml( + <container version="1.0"> + <handler id="test-handler" class={classOf[TestHandler].getName}> + <binding>http://*/TestHandler</binding> + </handler> + </container>)) + { container => + val response = container.handleRequest(new Request("http://foo/TestHandler")) + assertThat(response.getBodyAsString, is(TestHandler.RESPONSE)) + } + } + + @Test + def load_searcher_from_bundle() { + using(JDisc.fromPath(FileSystems.getDefault.getPath("src/test/app-packages/searcher-app"), Networking.disable)) + { container => + val result = container.search.process(ComponentSpecification.fromString("default"),new Query("?query=ignored")) + assertThat(result.hits().get(0).getField("title").toString, is("Heal the World!")) + } + } + + @Test + def document_types_can_be_accessed() { + using(new ApplicationBuilder().documentType("example", exampleDocument). + servicesXml(containerWithDocumentProcessing). + build()) + { application => + val container = application.getJDisc("jdisc") + val processing = container.documentProcessing() + assertThat(processing.getDocumentTypes.keySet(), hasItem("example")) + } + } + + @Test + def annotation_types_can_be_accessed() { + using(new ApplicationBuilder().documentType("example", + s""" + |search example { + | ${exampleDocument} + | annotation exampleAnnotation {} + |} + """.stripMargin). + servicesXml(containerWithDocumentProcessing). + build()) + { application => + val container = application.getJDisc("jdisc") + val processing = container.documentProcessing() + assertThat(processing.getAnnotationTypes.keySet(), hasItem("exampleAnnotation")) + } + } + + @Ignore //Enable this when static state has been removed. + @Test + def multiple_containers_can_be_run_in_parallel() { + def sendRequest(jdisc: JDisc) { + val response = jdisc.handleRequest(new Request("http://foo/TestHandler")) + assertThat(response.getBodyAsString, is(TestHandler.RESPONSE)) + } + + using(jdiscWithHttp()) { jdisc1 => + using(jdiscWithHttp()) { jdisc2 => + sendRequest(jdisc1) + sendRequest(jdisc2) + } + } + } +} + +object JDiscTest { + + def fromServicesXml(elem: Elem, networking: Networking = Networking.disable) = + JDisc.fromServicesXml(elem.toString(), networking) + + def using[T <: AutoCloseable, U](t: T)(f: T => U ) = { + try { + f(t) + } finally { + t.close() + } + } + + implicit def xmlToString(xml: Node): String = xml.toString() + + val containerWithDocumentProcessing = + <jdisc version="1.0"> + <http /> + <document-processing /> + </jdisc> + + val exampleDocument = + """ + |document example { + | + | field title type string { + | indexing: summary | index # How this field should be indexed + | weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature + | header + | } + |} + """.stripMargin + + + def jdiscWithHttp() = { + fromServicesXml( + <jdisc version="1.0"> + <handler id={classOf[TestHandler].getName} /> + <http> + <server id="main" port="9999" /> + </http> + </jdisc>) + } + + def getListenPort: Int = + Container.get.getServerProviderRegistry.allComponents().collectFirst { + case server: JettyHttpServer => server.getListenPort + } getOrElse { + throw new RuntimeException("No http server found") + } +} diff --git a/application/src/test/scala/com/yahoo/application/container/handlers/TestHandler.scala b/application/src/test/scala/com/yahoo/application/container/handlers/TestHandler.scala new file mode 100644 index 00000000000..acc1c74abfe --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/container/handlers/TestHandler.scala @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.handlers + +import com.yahoo.jdisc.handler.{ResponseDispatch, ResponseHandler, AbstractRequestHandler} +import TestHandler._ + +/** + * @author gjoranv + * @since 5.1.15 + */ + +class TestHandler extends AbstractRequestHandler { + def handleRequest(request:JDiscRequest, handler: ResponseHandler) = { + val writer = ResponseDispatch.newInstance(com.yahoo.jdisc.Response.Status.OK).connectFastWriter(handler) + writer.write(RESPONSE) + writer.close() + null + } +} +object TestHandler { + val RESPONSE = "Hello, World!" + type JDiscRequest = com.yahoo.jdisc.Request +} diff --git a/application/src/test/scala/com/yahoo/application/container/jersey/JerseyTest.scala b/application/src/test/scala/com/yahoo/application/container/jersey/JerseyTest.scala new file mode 100644 index 00000000000..de1ce3fcd27 --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/container/jersey/JerseyTest.scala @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.jersey + +import java.nio.file.{Path, Files, Paths} +import javax.ws.rs.core.UriBuilder + +import com.yahoo.application.Networking + +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase +import com.yahoo.vespa.scalalib.osgi.maven.ProjectBundleClassPaths +import com.yahoo.vespa.scalalib.osgi.maven.ProjectBundleClassPaths.BundleClasspathMapping +import org.apache.http.HttpResponse +import org.apache.http.client.methods.HttpGet +import org.apache.http.impl.client.DefaultHttpClient +import org.apache.http.util.EntityUtils +import org.junit.Assert._ +import org.junit.Test + +import com.yahoo.container.test.jars.jersey.{resources => jarResources} + +import org.hamcrest.CoreMatchers.is + +import com.yahoo.application.container.JDiscTest._ + +import scala.io.Source + +/** + * @author tonytv + */ +class JerseyTest { + type TestResourceClass = Class[_ <: TestResourceBase] + + val testJar = Paths.get("target/test-jars/jersey-resources.jar") + val testClassesDirectory = "target/test-classes" + val bundleSymbolicName = "myBundle" + + val classPathResources = Set( + classOf[resources.TestResource], + classOf[resources.nestedpackage1.NestedTestResource1], + classOf[resources.nestedpackage2.NestedTestResource2]) + + val jarFileResources = Set( + classOf[jarResources.TestResource], + classOf[com.yahoo.container.test.jars.jersey.resources.nestedpackage1.NestedTestResource1], + classOf[com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2]) + + @Test + def jersey_resources_on_classpath_can_be_invoked_from_application(): Unit = { + saveMainBundleClassPathMappings(testClassesDirectory) + + with_jersey_resources() { httpGetter => + assertResourcesResponds(classPathResources, httpGetter) + } + } + + @Test + def jersey_resources_in_provided_dependencies_can_be_invoked_from_application(): Unit = { + val providedDependency = BundleClasspathMapping(bundleSymbolicName, List(testClassesDirectory)) + + save(ProjectBundleClassPaths( + mainBundle = BundleClasspathMapping("main", List()), + providedDependencies = List(providedDependency))) + + with_jersey_resources() { httpGetter => + assertResourcesResponds(classPathResources, httpGetter) + } + } + + @Test + def jersey_resource_on_classpath_can_be_filtered_using_packages(): Unit = { + saveMainBundleClassPathMappings(testClassesDirectory) + + with_jersey_resources( + packagesToScan = List( + "com.yahoo.application.container.jersey.resources", + "com.yahoo.application.container.jersey.resources.nestedpackage1")) + { httpGetter => + val nestedResource2 = classOf[resources.nestedpackage2.NestedTestResource2] + + assertDoesNotRespond(nestedResource2, httpGetter) + assertResourcesResponds(classPathResources - nestedResource2, httpGetter) + } + } + + @Test + def jersey_resource_in_jar_can_be_invoked_from_application(): Unit = { + saveMainBundleJarClassPathMappings(testJar) + + with_jersey_resources() { httpGetter => + assertResourcesResponds(jarFileResources, httpGetter) + } + } + + @Test + def jersey_resource_in_jar_can_be_filtered_using_packages(): Unit = { + saveMainBundleJarClassPathMappings(testJar) + + with_jersey_resources( + packagesToScan = List( + "com.yahoo.container.test.jars.jersey.resources", + "com.yahoo.container.test.jars.jersey.resources.nestedpackage1")) + { httpGetter => + val nestedResource2 = classOf[com.yahoo.container.test.jars.jersey.resources.nestedpackage2.NestedTestResource2] + + assertDoesNotRespond(nestedResource2, httpGetter) + assertResourcesResponds(jarFileResources - nestedResource2, httpGetter) + } + } + + def with_jersey_resources(packagesToScan: List[String] = List())( f: HttpGetter => Unit): Unit = { + val packageElements = packagesToScan.map { p => <package>{p}</package>} + + using(fromServicesXml( + <services> + <jdisc version="1.0" id="default" jetty="true"> + <rest-api path="rest-api" jersey2="true"> + <components bundle={bundleSymbolicName}> + { packageElements } + </components> + </rest-api> + <http> + <server id="mainServer" port="0" /> + </http> + </jdisc> + </services>, + Networking.enable)) { jdisc => + + val client = new DefaultHttpClient + + def httpGetter(path: HttpPath) = { + client.execute(new HttpGet(s"http://localhost:$getListenPort/rest-api/${path.stripPrefix("/")}")) + } + + f(httpGetter) + } + } + + def assertResourcesResponds(resourceClasses: Traversable[TestResourceClass], httpGetter: HttpGetter): Unit = { + for (resource <- resourceClasses) { + val response = httpGetter(path(resource)) + assertThat(s"Failed sending response to $resource", response.getStatusLine.getStatusCode, is(200)) + + val content = Source.fromInputStream(response.getEntity.getContent).mkString + assertThat(content, is(TestResourceBase.content(resource))) + } + } + + def assertDoesNotRespond(resourceClass: TestResourceClass, httpGetter: HttpGetter): Unit = { + val response = httpGetter(path(resourceClass)) + assertThat(response.getStatusLine.getStatusCode, is(404)) + EntityUtils.consume(response.getEntity) + } + + def saveMainBundleJarClassPathMappings(jarFile: Path): Unit = { + assertTrue(s"Couldn't find file $jarFile, please remember to run mvn process-test-resources first.", Files.isRegularFile(jarFile)) + saveMainBundleClassPathMappings(jarFile.toAbsolutePath.toString) + } + + def saveMainBundleClassPathMappings(classPathElement: String): Unit = { + val mainBundleClassPathMappings = BundleClasspathMapping(bundleSymbolicName, List(classPathElement)) + save(ProjectBundleClassPaths(mainBundleClassPathMappings, providedDependencies = List())) + } + + def save(projectBundleClassPaths: ProjectBundleClassPaths): Unit = { + val path = Paths.get(testClassesDirectory).resolve(ProjectBundleClassPaths.classPathMappingsFileName) + ProjectBundleClassPaths.save(path, projectBundleClassPaths) + } + + def path(resourceClass: TestResourceClass) = { + UriBuilder.fromResource(resourceClass).build().toString + } + + type HttpPath = String + type HttpGetter = HttpPath => HttpResponse +} diff --git a/application/src/test/scala/com/yahoo/application/container/jersey/resources/TestResource.scala b/application/src/test/scala/com/yahoo/application/container/jersey/resources/TestResource.scala new file mode 100644 index 00000000000..a0c769721ab --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/container/jersey/resources/TestResource.scala @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.jersey.resources + +import javax.ws.rs.Path +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase + + +/** + * @author tonytv + */ +@Path("/test-resource") +class TestResource extends TestResourceBase diff --git a/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.scala b/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.scala new file mode 100644 index 00000000000..e6b0bb2de8e --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage1/NestedTestResource1.scala @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.jersey.resources.nestedpackage1 + +import javax.ws.rs.Path + +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase + +/** + * @author tonytv + */ +@Path("/nested-test-resource1") +class NestedTestResource1 extends TestResourceBase diff --git a/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.scala b/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.scala new file mode 100644 index 00000000000..7d4e715910d --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/container/jersey/resources/nestedpackage2/NestedTestResource2.scala @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.jersey.resources.nestedpackage2 + +import javax.ws.rs.Path + +import com.yahoo.container.test.jars.jersey.resources.TestResourceBase + +/** + * @author tonytv + */ +@Path("/nested-test-resource2") +class NestedTestResource2 extends TestResourceBase diff --git a/application/src/test/scala/com/yahoo/application/container/searchers/AddHitSearcher.scala b/application/src/test/scala/com/yahoo/application/container/searchers/AddHitSearcher.scala new file mode 100644 index 00000000000..a7427414315 --- /dev/null +++ b/application/src/test/scala/com/yahoo/application/container/searchers/AddHitSearcher.scala @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.searchers + +import com.yahoo.search.{Searcher, Result, Query} +import com.yahoo.search.searchchain.Execution +import com.yahoo.search.result.Hit + + +class AddHitSearcher extends Searcher { + + override def search(query: Query, execution: Execution) : Result = { + val result = execution.search(query) + result.hits().add(dummyHit) + + result + } + + private def dummyHit = { + val hit = new Hit("dummy") + hit.setField("title", getId.getName) + hit + } +} |