Publish

1 files changed, 355 insertions, 0 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java
new file mode 100644
index 00000000000..46823f30cf2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java
@@ -0,0 +1,355 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.config;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.pagetemplates.PageTemplate;
+import com.yahoo.search.pagetemplates.PageTemplateRegistry;
+import com.yahoo.search.pagetemplates.model.*;
+import com.yahoo.search.query.Sorting;
+import com.yahoo.text.XML;
+import org.w3c.dom.Element;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * Reads all page template XML files from a given directory (or list of readers).
+ * Instances of this are for single-thread usage only.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class PageTemplateXMLReader {
+
+    private static Logger logger=Logger.getLogger(PageTemplateXMLReader.class.getName());
+
+    /** The registry being constructed */
+    private PageTemplateRegistry registry;
+
+    /** XML elements by page id - available after phase 1. Needed for includes. */
+    private Map<ComponentId, Element> pageElementsByPageId=new LinkedHashMap<>();
+
+    /**
+     * Reads all page template xml files in a given directory.
+     *
+     * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML
+     */
+    public PageTemplateRegistry read(String directory) {
+        List<NamedReader> pageReaders=new ArrayList<>();
+        try {
+            File dir=new File(directory);
+            if ( !dir.isDirectory() ) throw new IllegalArgumentException("Could not read page templates: '" +
+                    directory + "' is not a valid directory.");
+
+            for (File file : sortFiles(dir)) {
+                if ( ! file.getName().endsWith(".xml")) continue;
+                pageReaders.add(new NamedReader(file.getName(),new FileReader(file)));
+            }
+
+            return read(pageReaders,true);
+        }
+        catch (IOException e) {
+            throw new IllegalArgumentException("Could not read page templates from '" + directory + "'",e);
+        }
+        finally {
+            for (NamedReader reader : pageReaders) {
+                try { reader.close(); } catch (IOException e) { }
+            }
+        }
+    }
+
+    /**
+     * Reads a single page template file.
+     *
+     * @throws RuntimeException if <code>fileName</code> is not a readable file, or if there is some error in the XML
+     */
+    public PageTemplate readFile(String fileName) {
+        NamedReader pageReader=null;
+        try {
+            File file=new File(fileName);
+            pageReader=new NamedReader(fileName,new FileReader(file));
+            String firstName=file.getName().substring(0,file.getName().length()-4);
+            return read(Collections.singletonList(pageReader),true).getComponent(firstName);
+        }
+        catch (IOException e) {
+            throw new IllegalArgumentException("Could not read the page template '" + fileName + "'",e);
+        }
+        finally {
+            if (pageReader!=null)
+                try { pageReader.close(); } catch (IOException e) { }
+        }
+    }
+
+    private List<File> sortFiles(File dir) {
+        ArrayList<File> files = new ArrayList<>();
+        files.addAll(Arrays.asList(dir.listFiles()));
+        Collections.sort(files);
+        return files;
+    }
+
+    /**
+     * Reads all page template xml files in a given list of readers. This is called from the Vespa configuration model.
+     *
+     * @param validateReaderNames should be set to true if the readers were created by files, not otherwise
+     * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML
+     */
+    public PageTemplateRegistry read(List<NamedReader> pageReaders,boolean validateReaderNames) {
+        // Initialize state
+        registry=new PageTemplateRegistry();
+
+        // Phase 1
+        pageElementsByPageId=createPages(pageReaders,validateReaderNames);
+        // Phase 2
+        readPages();
+        return registry;
+    }
+
+    private Map<ComponentId,Element> createPages(List<NamedReader> pageReaders,boolean validateReaderNames) {
+        Map<ComponentId,Element> pageElementsByPageId=new LinkedHashMap<>();
+        for (NamedReader reader : pageReaders) {
+            Element pageElement= XML.getDocument(reader).getDocumentElement();
+            if ( ! pageElement.getNodeName().equals("page")) {
+                logger.info("Ignoring '" + reader.getName() +
+                         "': Expected XML root element 'page' but was '" + pageElement.getNodeName() + "'");
+                continue;
+            }
+            String idString=pageElement.getAttribute("id");
+
+            if (idString==null || idString.isEmpty())
+                throw new IllegalArgumentException("Page template '" + reader.getName() + "' has no 'id' attribute in the root element");
+            ComponentId id=new ComponentId(idString);
+            if (validateReaderNames)
+                validateFileName(reader.getName(),id,"page template");
+            registry.register(new PageTemplate(id));
+            pageElementsByPageId.put(id,pageElement);
+        }
+        return pageElementsByPageId;
+    }
+
+    /** Throws an exception if the name is not corresponding to the id */
+    private void validateFileName(final String actualName,ComponentId id,String artifactName) {
+        String expectedCanonicalFileName=id.toFileName();
+        String fileName=new File(actualName).getName();
+        fileName=stripXmlEnding(fileName);
+        String canonicalFileName=ComponentId.fromFileName(fileName).toFileName();
+        if ( ! canonicalFileName.equals(expectedCanonicalFileName))
+            throw new IllegalArgumentException("The file name of " + artifactName + " '" + id +
+                                               "' must be '" + expectedCanonicalFileName + ".xml' but was '" + actualName + "'");
+    }
+
+    private String stripXmlEnding(String fileName) {
+        if (!fileName.endsWith(".xml"))
+            throw new IllegalArgumentException("'" + fileName + "' should have a .xml ending");
+        else
+            return fileName.substring(0,fileName.length()-4);
+    }
+
+    private void readPages() {
+        for (Map.Entry<ComponentId,Element> pageElement : pageElementsByPageId.entrySet()) {
+            try {
+                PageTemplate page=registry.getComponent(pageElement.getValue().getAttribute("id"));
+                readPageContent(pageElement.getValue(),page);
+            }
+            catch (IllegalArgumentException e) {
+                throw new IllegalArgumentException("Could not read page template '" + pageElement.getKey() + "'",e);
+            }
+        }
+    }
+
+    private void readPageContent(Element pageElement,PageTemplate page) {
+        if (page.isFrozen()) return; // Already read
+        Section rootSection=new Section(page.getId().toString());
+        readSection(pageElement,rootSection);
+        page.setSection(rootSection);
+        page.freeze();
+    }
+
+    /** Fills a section with attributes and sub-elements from a "section" or "page" element */
+    private Section readSection(Element sectionElement,Section section) {
+        section.setLayout(Layout.fromString(sectionElement.getAttribute("layout")));
+        section.setRegion(sectionElement.getAttribute("region"));
+        section.setOrder(Sorting.fromString(sectionElement.getAttribute("order")));
+        section.setMax(readOptionalNumber(sectionElement,"max"));
+        section.setMin(readOptionalNumber(sectionElement,"min"));
+        section.elements().addAll(readSourceAttribute(sectionElement));
+        section.elements().addAll(readPageElements(sectionElement));
+        return section;
+    }
+
+    /** Returns all page elements found under the given node */
+    private List<PageElement> readPageElements(Element parent) {
+        List<PageElement> pageElements=new ArrayList<>();
+        for (Element child : XML.getChildren(parent)) {
+            if (child.getNodeName().equals("include"))
+                pageElements.addAll(readInclude(child));
+            else
+                addIfNonNull(readPageElement(child),pageElements);
+        }
+        return pageElements;
+    }
+
+    private void addIfNonNull(PageElement pageElement,List<PageElement> pageElements) {
+        if (pageElement!=null)
+            pageElements.add(pageElement);
+    }
+
+    /** Reads the direct descendant elements of an include */
+    private List<PageElement> readInclude(Element element) {
+        PageTemplate included=registry.getComponent(element.getAttribute("idref"));
+        if (included==null)
+            throw new IllegalArgumentException("Could not find page template '" + element.getAttribute("idref"));
+        readPageContent(pageElementsByPageId.get(included.getId()),included);
+        return included.getSection().elements(Section.class);
+    }
+
+    /** Returns the page element corresponding to the given node, never null */
+    private PageElement readPageElement(Element child) {
+        if (child.getNodeName().equals("choice"))
+            return readChoice(child);
+        else if (child.getNodeName().equals("source"))
+            return readSource(child);
+        else if (child.getNodeName().equals("placeholder"))
+            return readPlaceholder(child);
+        else if (child.getNodeName().equals("section"))
+            return readSection(child,new Section(child.getAttribute("id")));
+        else if (child.getNodeName().equals("renderer"))
+            return readRenderer(child);
+        else if (child.getNodeName().equals("parameter"))
+            return null; // read elsewhere
+        throw new IllegalArgumentException("Unknown node type '" + child.getNodeName() + "'");
+    }
+
+    private List<Source> readSourceAttribute(Element sectionElement) {
+        List<Source> sources=new ArrayList<>();
+        String sourceAttributeString=sectionElement.getAttribute("source");
+        if (sourceAttributeString!=null) {
+            for (String sourceName : sourceAttributeString.split(" ")) {
+                if (sourceName.isEmpty()) continue;
+                if ("*".equals(sourceName))
+                    sources.add(Source.any);
+                else
+                    sources.add(new Source(sourceName));
+            }
+        }
+        return sources;
+    }
+
+    private Source readSource(Element sourceElement) {
+        Source source=new Source(sourceElement.getAttribute("name"));
+        source.setUrl(nullIfEmpty(sourceElement.getAttribute("url")));
+        source.renderers().addAll(readPageElements(sourceElement));
+        /*
+        source.renderers().addAll(readRenderers(XML.children(sourceElement,"renderer")));
+        readChoices(sourceElement,source);
+        */
+        source.parameters().putAll(readParameters(sourceElement));
+        return source;
+    }
+
+    private String nullIfEmpty(String s) {
+        if (s==null) return s;
+        s=s.trim();
+        if (s.isEmpty()) return null;
+        return s;
+    }
+
+    private Placeholder readPlaceholder(Element placeholderElement) {
+        return new Placeholder(placeholderElement.getAttribute("id"));
+    }
+
+    private Renderer readRenderer(Element rendererElement) {
+        Renderer renderer =new Renderer(rendererElement.getAttribute("name"));
+        renderer.setRendererFor(nullIfEmpty(rendererElement.getAttribute("for")));
+        renderer.parameters().putAll(readParameters(rendererElement));
+        return renderer;
+    }
+
+    private int readOptionalNumber(Element element,String attributeName) {
+        String attributeValue=element.getAttribute(attributeName);
+        try {
+            if (attributeValue.isEmpty()) return -1;
+            return Integer.parseInt(attributeValue);
+        }
+        catch (NumberFormatException e) { // Suppress original exception as it conveys no useful information
+            throw new IllegalArgumentException("'" + attributeName + "' in " + element + " must be a number, not '" + attributeValue + "'");
+        }
+    }
+
+    private AbstractChoice readChoice(Element choiceElement) {
+        String method=nullIfEmpty(choiceElement.getAttribute("method"));
+        if (XML.getChildren(choiceElement,"map").size()>0)
+            return readMapChoice(choiceElement,method);
+        else
+            return readNonMapChoice(choiceElement,method);
+    }
+
+    private MapChoice readMapChoice(Element choiceElement,String method) {
+        Element mapElement=XML.getChildren(choiceElement,"map").get(0);
+        MapChoice map=new MapChoice();
+        map.setMethod(method);
+
+        map.placeholderIds().addAll(readSpaceSeparatedAttribute("to",mapElement));
+        for (Element value : XML.getChildren(mapElement)) {
+            if ("item".equals(value.getNodeName()))
+                map.values().add(readPageElements(value));
+            else
+                map.values().add(Collections.singletonList(readPageElement(value)));
+        }
+        return map;
+    }
+
+    private Choice readNonMapChoice(Element choiceElement,String method) {
+        Choice choice=new Choice();
+        choice.setMethod(method);
+
+        for (Element alternative : XML.getChildren(choiceElement)) {
+            if (alternative.getNodeName().equals("alternative")) // Explicit alternative container
+                choice.alternatives().add(readPageElements(alternative));
+            else if (alternative.getNodeName().equals("include")) // Implicit include
+                choice.alternatives().add(readInclude(alternative));
+            else // Other implicit
+                choice.alternatives().add(Collections.singletonList(readPageElement(alternative)));
+        }
+        return choice;
+    }
+
+    /*
+    private void readChoices(Element sourceElement,Source source) {
+        for (Element choiceElement : XML.children(sourceElement,"choice")) {
+            for (Element alternative : XML.children(choiceElement)) {
+                if ("alternative".equals(alternative.getNodeName())) // Explicit alternative container
+                    source.renderer().alternatives().addAll(readRenderers(XML.children(alternative)));
+                else // Implicit alternative - yes implicit and explicit may be combined
+                    source.renderer().alternatives().addAll(readRenderers(Collections.singletonList(alternative)));
+            }
+        }
+    }
+    */
+
+    private Map<String,String> readParameters(Element containingElement) {
+        List<Element> parameterElements=XML.getChildren(containingElement,"parameter");
+        if (parameterElements.size()==0) return Collections.emptyMap(); // Shortcut
+
+        Map<String,String> parameters=new LinkedHashMap<>();
+        for (Element parameter : parameterElements) {
+            String key=parameter.getAttribute("name");
+            String value=XML.getValue(parameter);
+            parameters.put(key,value);
+        }
+        return parameters;
+    }
+
+    private List<String> readSpaceSeparatedAttribute(String attributeName, Element containingElement) {
+        List<String> values=new ArrayList<>();
+        String attributeString=nullIfEmpty(containingElement.getAttribute(attributeName));
+        if (attributeString!=null) {
+            for (String value : attributeString.split(" "))
+                values.add(value);
+        }
+        return values;
+    }
+
+}