diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo')
40 files changed, 597 insertions, 1126 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 8915c4b42f0..306c7c80577 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -23,31 +23,11 @@ import java.util.Set; */ public class Index { - public static class Attribute { - - private boolean tokenizedContent = false; - public final String name; - - public Attribute(String name) { - this.name = name; - } - - public boolean isTokenizedContent() { - return tokenizedContent; - } - - public void setTokenizedContent(boolean tokenizedContent) { - this.tokenizedContent = tokenizedContent; - } - } - /** The null index - don't use this for name lookups */ public static final Index nullIndex = new Index("(null)"); private final String name; - private String type; // TODO: Parse to a type object; do not expose this as a string - private final List<String> aliases = new ArrayList<>(); // The state resulting from adding commands to this (using addCommand) diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 37b0fd7ebfb..ccb6e1248b4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -4,12 +4,12 @@ package com.yahoo.prelude.cluster; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.QrConfig; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.prelude.fastsearch.FastSearcher; import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; @@ -66,7 +66,7 @@ public class ClusterSearcher extends Searcher { ClusterConfig clusterConfig, DocumentdbInfoConfig documentDbConfig, ComponentRegistry<Dispatcher> dispatchers, - FS4ResourcePool fs4ResourcePool, + QrConfig qrConfig, VipStatus vipStatus) { super(id); @@ -92,12 +92,12 @@ public class ClusterSearcher extends Searcher { } if (searchClusterConfig.indexingmode() == STREAMING) { - VdsStreamingSearcher searcher = vdsCluster(fs4ResourcePool.getServerId(), searchClusterIndex, + VdsStreamingSearcher searcher = vdsCluster(qrConfig.discriminator(), searchClusterIndex, searchClusterConfig, docSumParams, documentDbConfig); addBackendSearcher(searcher); vipStatus.addToRotation(searcher.getName()); } else { - FastSearcher searcher = searchDispatch(searchClusterIndex, searchClusterName, fs4ResourcePool.getServerId(), + FastSearcher searcher = searchDispatch(searchClusterIndex, searchClusterName, qrConfig.discriminator(), docSumParams, documentDbConfig, dispatchers); addBackendSearcher(searcher); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java deleted file mode 100644 index ed9eb72d7dd..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4ResourcePool.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - -import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.concurrent.ThreadFactoryFactory; -import com.yahoo.container.QrConfig; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * All users will get the same pool instance. - * - * @author baldersheim - */ -public class FS4ResourcePool extends AbstractComponent { - - private static final Logger logger = Logger.getLogger(FS4ResourcePool.class.getName()); - private static final AtomicInteger instanceCounter = new AtomicInteger(0); - private final String serverId; - private final int instanceId; - private final ExecutorService executor; - private final ScheduledExecutorService scheduledExecutor; - - @Inject - public FS4ResourcePool(QrConfig config) { - this(config.discriminator()); - } - - public FS4ResourcePool(String serverId) { - this.serverId = serverId; - instanceId = instanceCounter.getAndIncrement(); - String name = "FS4-" + instanceId; - executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(name)); - scheduledExecutor = Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory(name + ".scheduled")); - } - - /** Returns an unique identifier of the server this runs in */ - public String getServerId() { return serverId; } - public ExecutorService getExecutor() { return executor; } - public ScheduledExecutorService getScheduledExecutor() { return scheduledExecutor; } - - @Override - public void deconstruct() { - logger.log(Level.INFO, "Deconstructing FS4ResourcePool with id '" + instanceId + "'."); - super.deconstruct(); - executor.shutdown(); - scheduledExecutor.shutdown(); - try { - executor.awaitTermination(10, TimeUnit.SECONDS); - scheduledExecutor.awaitTermination(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - logger.warning("Executors failed terminating within timeout of 10 seconds : " + e); - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/FalseItem.java b/container-search/src/main/java/com/yahoo/prelude/query/FalseItem.java index 531a89312df..9abc6b2bdaa 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/FalseItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/FalseItem.java @@ -4,7 +4,7 @@ package com.yahoo.prelude.query; import java.nio.ByteBuffer; /** - * A query item which never matches. This is sometimes an useful output of query rewriting. + * A query item which never matches. This is sometimes a useful output of query rewriting. * * @author bratseth */ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java index 88bae76b26d..2bf20bf7c5a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java @@ -94,7 +94,7 @@ public class QueryCanonicalizer { if (composite instanceof RankItem || composite instanceof NotItem) { collapseLevels(composite, composite.getItemIterator()); // collapse the first item only } - else if (composite instanceof AndItem || composite instanceof OrItem) { + else if (composite instanceof AndItem || composite instanceof OrItem || composite instanceof WeakAndItem) { for (ListIterator<Item> i = composite.getItemIterator(); i.hasNext(); ) collapseLevels(composite, i); } @@ -106,10 +106,17 @@ public class QueryCanonicalizer { Item child = i.next(); if (child == null) return; if (child.getClass() != composite.getClass()) return; + if (child instanceof WeakAndItem && !equalWeakAndSettings((WeakAndItem)child, (WeakAndItem)composite)) return; i.remove(); moveChildren((CompositeItem) child, i); } - + + private static boolean equalWeakAndSettings(WeakAndItem a, WeakAndItem b) { + if ( ! a.getIndexName().equals(b.getIndexName())) return false; + if (a.getN() != b.getN()) return false; + return true; + } + private static void moveChildren(CompositeItem from, ListIterator<Item> toIterator) { for (ListIterator<Item> i = from.getItemIterator(); i.hasNext(); ) toIterator.add(i.next()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java index ac4e8b98b03..58bbcd7315c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SameElementItem.java @@ -50,6 +50,7 @@ public class SameElementItem extends NonReducibleCompositeItem { super.adding(item); //TODO See if we can require only SimpleIndexedItem instead of TermItem Validator.ensureInstanceOf("Child item", item, TermItem.class); + Validator.ensureNotInstanceOf("Child item", item, WordAlternativesItem.class); TermItem asTerm = (TermItem) item; Validator.ensureNonEmpty("Struct fieldname", asTerm.getIndexName()); Validator.ensureNonEmpty("Query term", asTerm.getIndexedString()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java index 8cce8fb5720..c5679e113f1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java @@ -99,13 +99,10 @@ public class WandItem extends WeightedSetItem { protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); - buffer.append(targetNumHits); - buffer.append(","); - buffer.append(scoreThreshold); - buffer.append(","); + buffer.append(targetNumHits).append(","); + buffer.append(scoreThreshold).append(","); buffer.append(thresholdBoostFactor); - buffer.append(")"); - buffer.append(" "); + buffer.append(") "); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 4fa2ed8b214..e8817a44133 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -18,16 +18,20 @@ import java.nio.ByteBuffer; */ public final class WeakAndItem extends NonReducibleCompositeItem { + /** The default N used if none is specified: 100 */ + public static final int defaultN = 100; + private int n; private String index; private int scoreThreshold = 0; - public ItemType getItemType() { - return ItemType.WEAK_AND; + /** Creates a WAND item with default N */ + public WeakAndItem() { + this(defaultN); } - public String getName() { - return "WAND"; + public WeakAndItem(int N) { + this("", N); } /** @@ -42,50 +46,37 @@ public final class WeakAndItem extends NonReducibleCompositeItem { this.n = n; this.index = (index == null) ? "" : index; } - public WeakAndItem(int N) { - this("", N); - } - /** Sets the index name of all subitems of this */ + @Override + public ItemType getItemType() { return ItemType.WEAK_AND; } + + @Override + public String getName() { return "WEAKAND"; } + + @Override public void setIndexName(String index) { String toSet = (index == null) ? "" : index; super.setIndexName(toSet); this.index = toSet; } - public String getIndexName() { - return index; - } + public String getIndexName() { return index; } /** Appends the heading of this string - <code>[getName()]([limit]) </code> */ + @Override protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); buffer.append(n); - buffer.append(")"); - buffer.append(" "); + buffer.append(") "); } - /** The default N used if none is specified: 100 */ - public static final int defaultN = 100; + public int getN() { return n; } - /** Creates a WAND item with default N */ - public WeakAndItem() { - this(defaultN); - } - - public int getN() { - return n; - } - - public void setN(int N) { - this.n = N; - } + public void setN(int N) { this.n = N; } @Deprecated // TODO: Remove on Vespa 8 - public int getScoreThreshold() { - return scoreThreshold; - } + public int getScoreThreshold() { return scoreThreshold; } /** * Noop. @@ -93,9 +84,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { * @deprecated has no effect */ @Deprecated // TODO: Remove on Vespa 8 - public void setScoreThreshold(int scoreThreshold) { - this.scoreThreshold = scoreThreshold; - } + public void setScoreThreshold(int scoreThreshold) { this.scoreThreshold = scoreThreshold; } @Override protected void encodeThis(ByteBuffer buffer) { @@ -111,9 +100,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { } @Override - public int hashCode() { - return super.hashCode() + 31 * n; - } + public int hashCode() { return super.hashCode() + 31 * n; } /** Returns whether this item is of the same class and contains the same state as the given item. */ @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java index 2c017410109..d2df2aa6c89 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java @@ -19,7 +19,6 @@ import com.yahoo.compress.IntegerCompressor; public class WordAlternativesItem extends TermItem { private List<Alternative> alternatives; - private int maxIndex; public static final class Alternative { @@ -48,7 +47,6 @@ public class WordAlternativesItem extends TermItem { public void setAlternatives(Collection<Alternative> terms) { this.alternatives = uniqueAlternatives(terms); - setMaxIndex(); } private static ImmutableList<Alternative> uniqueAlternatives(Collection<Alternative> terms) { @@ -67,27 +65,6 @@ public class WordAlternativesItem extends TermItem { return ImmutableList.copyOf(uniqueTerms); } - private void setMaxIndex() { - int maxIndex = 0; - int currentIndex = 0; - double maxScore = 0.0d; - boolean first = true; - for (Alternative val : this.alternatives) { - if (first) { - first = false; - maxIndex = 0; - maxScore = val.exactness; - } else { - if (val.exactness > maxScore) { - maxScore = val.exactness; - maxIndex = currentIndex; - } - } - ++currentIndex; - } - this.maxIndex = maxIndex; - } - @Override public String stringValue() { StringBuilder builder = new StringBuilder(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index 74a993b0413..8b878417912 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -147,10 +147,10 @@ public class AdvancedParser extends StructuredParser { return equiv; } return topLevelItem; - } else if (isTheWord("wand", item)) { + } else if (isTheWord("wand", item) || isTheWord("weakand", item)) { int n = consumeNumericArgument(); if (n == 0) - n=WeakAndItem.defaultN; + n = WeakAndItem.defaultN; if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n != ((WeakAndItem)topLevelItem).getN()) { WeakAndItem wand = new WeakAndItem(); wand.setN(n); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java index 5a936d42ccc..329e886c3d3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java @@ -186,6 +186,7 @@ public class QueryRewrite { } else if ((item instanceof AndItem) || (item instanceof NearItem)) { return Recall.RECALLS_NOTHING; } else if (item instanceof RankItem) { + if (i == 0) return Recall.RECALLS_NOTHING; item.removeItem(i); } else { throw new UnsupportedOperationException(item.getClass().getName()); diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 941015b2fae..24b631f1773 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -5,9 +5,9 @@ import com.yahoo.component.chain.dependencies.Before; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.container.protect.Error; import com.yahoo.jdisc.Metric; -import java.util.logging.Level; -import com.yahoo.metrics.simple.MetricSettings; +import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.metrics.simple.MetricSettings; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -29,7 +29,18 @@ import java.util.PriorityQueue; import java.util.Queue; import java.util.logging.Level; -import static com.yahoo.container.protect.Error.*; +import static com.yahoo.container.protect.Error.BACKEND_COMMUNICATION_ERROR; +import static com.yahoo.container.protect.Error.EMPTY_DOCUMENTS; +import static com.yahoo.container.protect.Error.ERROR_IN_PLUGIN; +import static com.yahoo.container.protect.Error.ILLEGAL_QUERY; +import static com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; +import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; +import static com.yahoo.container.protect.Error.INVALID_QUERY_TRANSFORMATION; +import static com.yahoo.container.protect.Error.NO_BACKENDS_IN_SERVICE; +import static com.yahoo.container.protect.Error.RESULT_HAS_ERRORS; +import static com.yahoo.container.protect.Error.SERVER_IS_MISCONFIGURED; +import static com.yahoo.container.protect.Error.TIMEOUT; +import static com.yahoo.container.protect.Error.UNSPECIFIED; /** @@ -236,7 +247,7 @@ public class StatisticsSearcher extends Searcher { incrQueryCount(metricContext); logQuery(query); - long start_ns = System.nanoTime(); // Start time, in nanoseconds. + long start_ns = getStartNanoTime(query); qps(metricContext); Result result; //handle exceptions thrown below in searchers @@ -428,5 +439,15 @@ public class StatisticsSearcher extends Searcher { } } + /** + * Returns the relative start time from request was received by jdisc + */ + private static long getStartNanoTime(Query query) { + return Optional.ofNullable(query.getHttpRequest()) + .flatMap(httpRequest -> Optional.ofNullable(httpRequest.getJDiscRequest())) + .map(HttpRequest::relativeCreatedAtNanoTime) + .orElseGet(System::nanoTime); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java index 515d6249fd8..9329f4a6819 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java @@ -12,6 +12,7 @@ import java.util.function.BiConsumer; * @author ollivir */ public abstract class CloseableInvoker implements Closeable { + protected abstract void release(); private BiConsumer<Boolean, Long> teardown = null; @@ -35,4 +36,5 @@ public abstract class CloseableInvoker implements Closeable { } release(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 626cf087aca..9b92a78a7c9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -65,7 +65,7 @@ public class Dispatcher extends AbstractComponent { /** A model of the search cluster this dispatches to */ private final SearchCluster searchCluster; - private final ClusterMonitor clusterMonitor; + private final ClusterMonitor<Node> clusterMonitor; private final LoadBalancer loadBalancer; @@ -108,7 +108,7 @@ public class Dispatcher extends AbstractComponent { } /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(ClusterMonitor clusterMonitor, + protected Dispatcher(ClusterMonitor<Node> clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, @@ -125,12 +125,7 @@ public class Dispatcher extends AbstractComponent { this.metricContext = metric.createContext(null); this.maxHitsPerNode = dispatchConfig.maxHitsPerNode(); searchCluster.addMonitoring(clusterMonitor); - Thread warmup = new Thread(new Runnable() { - @Override - public void run() { - warmup(dispatchConfig.warmuptime()); - } - }); + Thread warmup = new Thread(() -> warmup(dispatchConfig.warmuptime())); warmup.start(); try { while ( ! searchCluster.hasInformationAboutAllNodes()) { @@ -139,20 +134,17 @@ public class Dispatcher extends AbstractComponent { warmup.join(); } catch (InterruptedException e) {} - /* - * No we have information from all nodes and a ping iteration has completed. - * Instead of waiting until next ping interval to update coverage and group state, - * we should compute the state ourselves, so that when the dispatcher is ready the state - * of its groups are also known. - */ + // Now we have information from all nodes and a ping iteration has completed. + // Instead of waiting until next ping interval to update coverage and group state, + // we should compute the state ourselves, so that when the dispatcher is ready the state + // of its groups are also known. searchCluster.pingIterationCompleted(); } - /* - Will run important code in order to trigger JIT compilation and avoid cold start issues. - Currently warms up lz4 compression code. + /** + * Will run important code in order to trigger JIT compilation and avoid cold start issues. + * Currently warms up lz4 compression code. */ - private static long warmup(double seconds) { return new Compressor().warmup(seconds); } @@ -164,7 +156,7 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { - /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */ + // The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. clusterMonitor.shutdown(); invokerFactory.release(); } @@ -212,7 +204,7 @@ public class Dispatcher extends AbstractComponent { return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), - Arrays.asList(node), + List.of(node), true, maxHitsPerNode) .orElseThrow(() -> new IllegalStateException("Could not dispatch directly to " + node)); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java index dd4c4494ac5..8b7714aaf3b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/FillInvoker.java @@ -10,7 +10,8 @@ import com.yahoo.search.Result; * @author ollivir */ public abstract class FillInvoker extends CloseableInvoker { - /** Retrieve document summaries for the unfilled hits in the given {@link Result} */ + + /** Retrieves document summaries for the unfilled hits in the given {@link Result} */ public void fill(Result result, String summaryClass) { sendFillRequest(result, summaryClass); getFillResults(result, summaryClass); @@ -19,4 +20,5 @@ public abstract class FillInvoker extends CloseableInvoker { protected abstract void getFillResults(Result result, String summaryClass); protected abstract void sendFillRequest(Result result, String summaryClass); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 036592dcf23..adf7368faa2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -235,17 +235,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return nextAdaptive; } - private String dbg(LeanHit hit) { - var buf = new StringBuilder(); - buf.append("LeanHit["); - if (hit.hasSortData()) buf.append("hasSortData,"); - buf.append("relevance=").append(hit.getRelevance()); - buf.append(",partId=").append(hit.getPartId()); - buf.append(",distributionKey=").append(hit.getDistributionKey()); - buf.append("]"); - return buf.toString(); - } - private List<LeanHit> mergeResult(Result result, InvokerResult partialResult, List<LeanHit> current) { collectCoverage(partialResult.getResult().getCoverage(true)); @@ -382,4 +371,5 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM // For testing Collection<SearchInvoker> invokers() { return invokers; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index f65e0e43757..1bcb640e3a5 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -35,7 +35,7 @@ public abstract class InvokerFactory { public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); /** - * Create a {@link SearchInvoker} for a list of content nodes. + * Creates a {@link SearchInvoker} for a list of content nodes. * * @param searcher the searcher processing the query * @param query the search query being processed @@ -79,7 +79,7 @@ public abstract class InvokerFactory { success.add(node); } } - if ( ! searchCluster.isPartialGroupCoverageSufficient(groupId, success) && !acceptIncompleteCoverage) { + if ( ! searchCluster.isPartialGroupCoverageSufficient(success) && !acceptIncompleteCoverage) { return Optional.empty(); } if (invokers.size() == 0) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java index 94c347a6927..2723429c0cf 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java @@ -12,14 +12,19 @@ import java.util.List; /** * Wraps a Result and a flat, skinny hit list + * + * @author baldersheim */ public class InvokerResult { + private final Result result; private final List<LeanHit> leanHits; + public InvokerResult(Result result) { this.result = result; this.leanHits = Collections.emptyList(); } + public InvokerResult(Query query, int expectedHits) { result = new Result(query); leanHits = new ArrayList<>(expectedHits); @@ -32,6 +37,7 @@ public class InvokerResult { public List<LeanHit> getLeanHits() { return leanHits; } + void complete() { Query query = result.getQuery(); Sorting sorting = query.getRanking().getSorting(); @@ -47,4 +53,5 @@ public class InvokerResult { } leanHits.clear(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 8a90557fa3b..df8fb2f29fa 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -4,7 +4,11 @@ package com.yahoo.search.dispatch; import java.util.Arrays; +/** + * @author baldersheim + */ public class LeanHit implements Comparable<LeanHit> { + private final byte [] gid; private final double relevance; private final byte [] sortData; @@ -21,6 +25,7 @@ public class LeanHit implements Comparable<LeanHit> { this.partId = partId; this.distributionKey = distributionKey; } + public double getRelevance() { return relevance; } public byte [] getGid() { return gid; } public byte [] getSortData() { return sortData; } @@ -49,4 +54,5 @@ public class LeanHit implements Comparable<LeanHit> { int vr = (int) right[i] & 0xFF; return vl - vr; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 05e1ea6e2f9..ebde2ffc611 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -43,8 +43,8 @@ public class LoadBalancer { } /** - * Select and allocate the search cluster group which is to be used for the next search query. Callers <b>must</b> call - * {@link #releaseGroup} symmetrically for each taken allocation. + * Select and allocate the search cluster group which is to be used for the next search query. + * Callers <b>must</b> call {@link #releaseGroup} symmetrically for each taken allocation. * * @param rejectedGroups if not null, the load balancer will only return groups with IDs not in the set * @return the node group to target, or <i>empty</i> if the internal dispatch logic cannot be used @@ -76,7 +76,7 @@ public class LoadBalancer { synchronized (this) { for (GroupStatus sched : scoreboard) { if (sched.group.id() == group.id()) { - sched.release(success, (double) searchTimeMs / 1000.0); + sched.release(success, searchTimeMs / 1000.0); break; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java b/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java index c2e81d43677..3ebd21fa18a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/ResponseMonitor.java @@ -9,5 +9,7 @@ package com.yahoo.search.dispatch; * @author ollivir */ public interface ResponseMonitor<T> { + void responseAvailable(T from); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index 256759360f7..7dbc2e98759 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -61,4 +61,5 @@ public class SearchErrorInvoker extends SearchInvoker { protected void setMonitor(ResponseMonitor<SearchInvoker> monitor) { this.monitor = monitor; } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java index 7937be50813..f6480f80c01 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java @@ -28,15 +28,11 @@ import java.util.stream.IntStream; public class SearchPath { /** - * Parse the search path and select nodes from the given cluster based on it. + * Parses the search path and select nodes from the given cluster based on it. * - * @param searchPath - * unparsed search path expression (see: model.searchPath in Search - * API reference) - * @param cluster - * the search cluster from which nodes are selected - * @throws InvalidSearchPathException - * if the searchPath is malformed + * @param searchPath unparsed search path expression (see: model.searchPath in Search API reference) + * @param cluster the search cluster from which nodes are selected + * @throws InvalidSearchPathException if the searchPath is malformed * @return list of nodes chosen with the search path, or an empty list in which * case some other node selection logic should be used */ diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java index aef1ef2f498..315dfdd4320 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java @@ -6,9 +6,11 @@ import org.apache.commons.math3.distribution.TDistribution; /** * Use StudentT distribution and estimate how many hits you need from each partition * to to get the globally top-k documents with the desired probability + * * @author baldersheim */ public class TopKEstimator { + private final TDistribution studentT; private final double defaultP; private final boolean estimate; @@ -19,9 +21,11 @@ public class TopKEstimator { private static boolean needEstimate(double p) { return (0.0 < p) && (p < 1.0); } + TopKEstimator(double freedom, double defaultProbability) { this(freedom, defaultProbability, 0.0); } + public TopKEstimator(double freedom, double defaultProbability, double skewFactor) { this.studentT = new TDistribution(null, freedom); defaultP = defaultProbability; @@ -32,36 +36,44 @@ public class TopKEstimator { defaultCumulativeProbability[i] = computeCumulativeProbability(i+MIN_N, defaultP); } } + private double inverseCumulativeProbability(int n, double p) { if (p == defaultP && (n >= MIN_N) && (n < defaultCumulativeProbability.length + MIN_N)) { return defaultCumulativeProbability[n - MIN_N]; } return computeCumulativeProbability(n, p); } + private double computeCumulativeProbability(int n, double p) { double p_inverse = 1 - (1 - p)/computeN(n); return studentT.inverseCumulativeProbability(p_inverse); } + private double computeN(double n) { double p_max = (1 + skewFactor)/n; return Math.max(1, 1/p_max); } + double estimateExactK(double k, int n_i, double p) { double n = computeN(n_i); double variance = k * 1/n * (1 - 1/n); return k/n + inverseCumulativeProbability(n_i, p) * Math.sqrt(variance); } + double estimateExactK(double k, int n) { return estimateExactK(k, n, defaultP); } + public int estimateK(int k, int n) { return (estimate && (n >= MIN_N)) ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, defaultP))) : k; } + public int estimateK(int k, int n, double p) { return (needEstimate(p) && (n >= MIN_N)) ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, p))) : k; } } + diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 6dc01f34571..250524fadf2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -196,7 +196,7 @@ public class ProtobufSerialization { result.getResult().setTotalHitCount(protobuf.getTotalHitCount()); result.getResult().setCoverage(convertToCoverage(protobuf)); - var haveGrouping = protobuf.getGroupingBlob() != null && !protobuf.getGroupingBlob().isEmpty(); + var haveGrouping = ! protobuf.getGroupingBlob().isEmpty(); if (haveGrouping) { BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(protobuf.getGroupingBlob().asReadOnlyByteBuffer())); int cnt = buf.getInt(null); @@ -219,7 +219,7 @@ public class ProtobufSerialization { } var slimeTrace = protobuf.getSlimeTrace(); - if (slimeTrace != null && !slimeTrace.isEmpty()) { + if ( ! slimeTrace.isEmpty()) { var traces = new Value.ArrayValue(); traces.add(new SlimeAdapter(BinaryFormat.decode(slimeTrace.toByteArray()).get())); query.trace(traces, query.getTraceLevel()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java index 7d9b3ca1034..01e3ec3ca2b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java @@ -8,12 +8,16 @@ import com.yahoo.search.dispatch.searchcluster.Pinger; import com.yahoo.search.dispatch.searchcluster.PongHandler; public class RpcPingFactory implements PingFactory { + private final RpcResourcePool rpcResourcePool; + public RpcPingFactory(RpcResourcePool rpcResourcePool) { this.rpcResourcePool = rpcResourcePool; } + @Override public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { return new RpcPing(node, monitor, rpcResourcePool, pongHandler); } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index 341b9b2bce3..8a17be8102e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -38,6 +38,7 @@ import java.util.logging.Logger; * @author ollivir */ public class RpcProtobufFillInvoker extends FillInvoker { + private static final String RPC_METHOD = "vespa.searchprotocol.getDocsums"; private static final Logger log = Logger.getLogger(RpcProtobufFillInvoker.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index 746461630dd..c3d072b8db6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -26,6 +26,7 @@ import java.util.Random; * @author ollivir */ public class RpcResourcePool extends AbstractComponent { + /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 4c0b77207d5..20b11efb470 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -102,9 +102,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe ProtobufResponse protobufResponse = response.response().get(); CompressionType compression = CompressionType.valueOf(protobufResponse.compression()); byte[] payload = resourcePool.compressor().decompress(protobufResponse.compressedPayload(), compression, protobufResponse.uncompressedSize()); - var result = ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); - - return result; + return ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index e5066797b06..dca5892e0e7 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -92,7 +92,7 @@ public class Group { } @Override - public String toString() { return "search group " + id; } + public String toString() { return "group " + id; } @Override public int hashCode() { return id; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index 8f465070de4..9807a978647 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -71,7 +71,7 @@ public class Node { } /** Updates the active documents on this node */ - void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } + public void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } /** Returns the active documents on this node. If unknown, 0 is returned. */ long getActiveDocuments() { return activeDocuments.get(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index 2e07d8d61e6..3b9e9573367 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -3,7 +3,9 @@ package com.yahoo.search.dispatch.searchcluster; import com.yahoo.search.cluster.ClusterMonitor; - +/** + * @author ollivir + */ public interface PingFactory { Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java index b4a7ccbf98c..681a7d0af2c 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -8,5 +8,7 @@ package com.yahoo.search.dispatch.searchcluster; * @author baldersheim */ public interface Pinger { + void ping(); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java index 1b39f14fd86..c39426e9d76 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java @@ -9,5 +9,7 @@ import com.yahoo.prelude.Pong; * @author baldersheim */ public interface PongHandler { + void handle(Pong pong); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index b3b2c23e7dc..ce834b108db 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; +import com.google.common.math.Quantiles; import com.yahoo.container.handler.VipStatus; import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; @@ -13,6 +14,8 @@ import com.yahoo.search.cluster.NodeManager; import com.yahoo.search.dispatch.TopKEstimator; import com.yahoo.vespa.config.search.DispatchConfig; +import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -311,33 +314,33 @@ public class SearchCluster implements NodeManager<Node> { boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.getActiveDocuments(), group.getActiveDocuments()); - trackGroupCoverageChanges(0, group, sufficientCoverage, group.getActiveDocuments()); + trackGroupCoverageChanges(group, sufficientCoverage, group.getActiveDocuments()); } private void pingIterationCompletedMultipleGroups() { - int numGroups = orderedGroups().size(); - // Update active documents per group and use it to decide if the group should be active - long[] activeDocumentsInGroup = new long[numGroups]; - long sumOfActiveDocuments = 0; - for(int i = 0; i < numGroups; i++) { - Group group = orderedGroups().get(i); - group.aggregateNodeValues(); - activeDocumentsInGroup[i] = group.getActiveDocuments(); - sumOfActiveDocuments += activeDocumentsInGroup[i]; - } - + aggregateNodeValues(); + long medianDocuments = medianDocumentsPerGroup(); boolean anyGroupsSufficientCoverage = false; - for (int i = 0; i < numGroups; i++) { - Group group = orderedGroups().get(i); - long activeDocuments = activeDocumentsInGroup[i]; - long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); - boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), activeDocuments, averageDocumentsInOtherGroups); + for (Group group : orderedGroups()) { + boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), + group.getActiveDocuments(), + medianDocuments); anyGroupsSufficientCoverage = anyGroupsSufficientCoverage || sufficientCoverage; updateSufficientCoverage(group, sufficientCoverage); - trackGroupCoverageChanges(i, group, sufficientCoverage, averageDocumentsInOtherGroups); + trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments); } } + private void aggregateNodeValues() { + orderedGroups().forEach(Group::aggregateNodeValues); + } + + private long medianDocumentsPerGroup() { + if (orderedGroups().isEmpty()) return 0; + var activeDocuments = orderedGroups().stream().map(Group::getActiveDocuments).collect(Collectors.toList()); + return (long)Quantiles.median().compute(activeDocuments); + } + /** * Update statistics after a round of issuing pings. * Note that this doesn't wait for pings to return, so it will typically accumulate data from @@ -353,10 +356,9 @@ public class SearchCluster implements NodeManager<Node> { } } - private boolean isGroupCoverageSufficient(int workingNodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { - double documentCoverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; - - if (averageDocumentsInOtherGroups > 0 && documentCoverage < dispatchConfig.minActivedocsPercentage()) + private boolean isGroupCoverageSufficient(int workingNodesInGroup, long activeDocuments, long medianDocuments) { + double documentCoverage = 100.0 * (double) activeDocuments / medianDocuments; + if (medianDocuments > 0 && documentCoverage < dispatchConfig.minActivedocsPercentage()) return false; if ( ! isGroupNodeCoverageSufficient(workingNodesInGroup)) @@ -380,56 +382,33 @@ public class SearchCluster implements NodeManager<Node> { /** * Calculate whether a subset of nodes in a group has enough coverage */ - public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) { - if (orderedGroups().size() == 1) { - boolean sufficient = nodes.size() >= wantedGroupSize() - dispatchConfig.maxNodesDownPerGroup(); - return sufficient; - } - - if (knownGroupId.isEmpty()) { - return false; - } - int groupId = knownGroupId.getAsInt(); - Group group = groups().get(groupId); - if (group == null) { - return false; - } - long sumOfActiveDocuments = 0; - int otherGroups = 0; - for (Group g : orderedGroups()) { - if (g.id() != groupId) { - sumOfActiveDocuments += g.getActiveDocuments(); - otherGroups++; - } - } - long activeDocuments = 0; - for (Node n : nodes) { - activeDocuments += n.getActiveDocuments(); - } - long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups; - return isGroupCoverageSufficient(nodes.size(), activeDocuments, averageDocumentsInOtherGroups); + public boolean isPartialGroupCoverageSufficient(List<Node> nodes) { + if (orderedGroups().size() == 1) + return nodes.size() >= wantedGroupSize() - dispatchConfig.maxNodesDownPerGroup(); + long activeDocuments = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); + return isGroupCoverageSufficient(nodes.size(), activeDocuments, medianDocumentsPerGroup()); } - private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { + private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments) { if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. boolean changed = group.isFullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { nextLogTime = System.currentTimeMillis() + 30 * 1000; int requiredNodes = group.nodes().size() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { - log.info(() -> String.format("Cluster %s: Group %d is now good again (%d/%d active docs, coverage %d/%d)", - clusterId, index, group.getActiveDocuments(), averageDocuments, - group.workingNodes(), group.nodes().size())); + log.info("Cluster " + clusterId + ": " + group + " has full coverage. " + + "Active documents: " + group.getActiveDocuments() + "/" + medianDocuments + ", " + + "working nodes: " + group.workingNodes() + "/" + group.nodes().size()); } else { - StringBuilder missing = new StringBuilder(); + StringBuilder unresponsive = new StringBuilder(); for (var node : group.nodes()) { - if (node.isWorking() != Boolean.TRUE) { - missing.append('\n').append(node); - } + if (node.isWorking() != Boolean.TRUE) + unresponsive.append('\n').append(node); } - log.warning(() -> String.format("Cluster %s: Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", - clusterId, index, group.workingNodes(), group.nodes().size(), requiredNodes, - group.getActiveDocuments(), averageDocuments, missing)); + log.warning("Cluster " + clusterId + ": " + group + " has reduced coverage: " + + "Active documents: " + group.getActiveDocuments() + "/" + medianDocuments + ", " + + "working nodes: " + group.workingNodes() + "/" + group.nodes().size() + " required " + requiredNodes + + ", unresponsive nodes: " + (unresponsive.toString().isEmpty() ? " none" : unresponsive)); } } } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 2b27f60ef73..65dc1052a78 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -11,14 +11,18 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.ContainerHttpConfig; import com.yahoo.container.handler.threadpool.ContainerThreadPool; +import com.yahoo.container.jdisc.HttpMethodAclMapping; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.RequestHandlerSpec; import com.yahoo.container.jdisc.VespaHeaders; import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; +import com.yahoo.container.jdisc.AclMapping; +import com.yahoo.container.jdisc.RequestView; import com.yahoo.language.Linguistics; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; @@ -103,6 +107,9 @@ public class SearchHandler extends LoggingRequestHandler { private final AtomicLong numRequestsLeftToTrace; + private final static RequestHandlerSpec REQUEST_HANDLER_SPEC = RequestHandlerSpec.builder() + .withAclMapping(SearchHandler.aclRequestMapper()).build(); + private final class MeanConnections implements Callback { @Override @@ -631,6 +638,16 @@ public class SearchHandler extends LoggingRequestHandler { }); } + @Override + public RequestHandlerSpec requestHandlerSpec() { + return REQUEST_HANDLER_SPEC; + } + + private static AclMapping aclRequestMapper() { + return HttpMethodAclMapping.standard() + .override(com.yahoo.jdisc.http.HttpRequest.Method.POST, AclMapping.Action.READ) + .build(); + } } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index 9e53f9d8ea9..94b9bf6ce65 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -4,7 +4,6 @@ package com.yahoo.search.query.parser; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.prelude.query.parser.SpecialTokens; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java index 25488aa7bbc..3aa9e59003d 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java @@ -31,8 +31,7 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { public boolean shouldLowercase(WordItem word, IndexFacts.Session indexFacts) { if (word.isLowercased()) return false; - Index index = indexFacts.getIndex(word.getIndexName()); - return index.isLowercase() || index.isAttribute(); + return indexFacts.getIndex(word.getIndexName()).isLowercase(); } @Override @@ -41,8 +40,7 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { StringBuilder sb = new StringBuilder(); sb.append(commonPath).append(".").append(word.getIndexName()); - Index index = indexFacts.getIndex(sb.toString()); - return index.isLowercase() || index.isAttribute(); + return indexFacts.getIndex(sb.toString()).isLowercase(); } } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index c4f850307ae..0a87ad7ec2b 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -65,6 +65,8 @@ import java.util.Set; import java.util.concurrent.Executor; import java.util.function.LongSupplier; +import static com.fasterxml.jackson.databind.SerializationFeature.FLUSH_AFTER_WRITE_VALUE; + /** * JSON renderer for search results. * @@ -147,7 +149,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { * @return an object mapper for the internal JsonFactory */ protected static ObjectMapper createJsonCodec() { - return new ObjectMapper(); + return new ObjectMapper().disable(FLUSH_AFTER_WRITE_VALUE); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index bcd5822a6f6..d3f07bae428 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -14,76 +14,46 @@ import com.yahoo.search.yql.yqlplusParser.AnnotateExpressionContext; import com.yahoo.search.yql.yqlplusParser.ArgumentContext; import com.yahoo.search.yql.yqlplusParser.ArgumentsContext; import com.yahoo.search.yql.yqlplusParser.ArrayLiteralContext; -import com.yahoo.search.yql.yqlplusParser.ArrayTypeContext; import com.yahoo.search.yql.yqlplusParser.Call_sourceContext; import com.yahoo.search.yql.yqlplusParser.ConstantArrayContext; import com.yahoo.search.yql.yqlplusParser.ConstantExpressionContext; import com.yahoo.search.yql.yqlplusParser.ConstantMapExpressionContext; import com.yahoo.search.yql.yqlplusParser.ConstantPropertyNameAndValueContext; -import com.yahoo.search.yql.yqlplusParser.Delete_statementContext; import com.yahoo.search.yql.yqlplusParser.DereferencedExpressionContext; import com.yahoo.search.yql.yqlplusParser.EqualityExpressionContext; import com.yahoo.search.yql.yqlplusParser.ExpressionContext; -import com.yahoo.search.yql.yqlplusParser.FallbackContext; import com.yahoo.search.yql.yqlplusParser.Field_defContext; -import com.yahoo.search.yql.yqlplusParser.Field_names_specContext; -import com.yahoo.search.yql.yqlplusParser.Field_values_group_specContext; -import com.yahoo.search.yql.yqlplusParser.Field_values_specContext; import com.yahoo.search.yql.yqlplusParser.IdentContext; -import com.yahoo.search.yql.yqlplusParser.Import_listContext; -import com.yahoo.search.yql.yqlplusParser.Import_statementContext; import com.yahoo.search.yql.yqlplusParser.InNotInTargetContext; -import com.yahoo.search.yql.yqlplusParser.Insert_sourceContext; -import com.yahoo.search.yql.yqlplusParser.Insert_statementContext; -import com.yahoo.search.yql.yqlplusParser.Insert_valuesContext; -import com.yahoo.search.yql.yqlplusParser.JoinExpressionContext; -import com.yahoo.search.yql.yqlplusParser.Join_exprContext; import com.yahoo.search.yql.yqlplusParser.LimitContext; import com.yahoo.search.yql.yqlplusParser.Literal_elementContext; import com.yahoo.search.yql.yqlplusParser.Literal_listContext; import com.yahoo.search.yql.yqlplusParser.LogicalANDExpressionContext; import com.yahoo.search.yql.yqlplusParser.LogicalORExpressionContext; import com.yahoo.search.yql.yqlplusParser.MapExpressionContext; -import com.yahoo.search.yql.yqlplusParser.MapTypeContext; -import com.yahoo.search.yql.yqlplusParser.Merge_componentContext; -import com.yahoo.search.yql.yqlplusParser.Merge_statementContext; -import com.yahoo.search.yql.yqlplusParser.ModuleIdContext; -import com.yahoo.search.yql.yqlplusParser.ModuleNameContext; import com.yahoo.search.yql.yqlplusParser.MultiplicativeExpressionContext; import com.yahoo.search.yql.yqlplusParser.Namespaced_nameContext; -import com.yahoo.search.yql.yqlplusParser.Next_statementContext; import com.yahoo.search.yql.yqlplusParser.OffsetContext; import com.yahoo.search.yql.yqlplusParser.OrderbyContext; import com.yahoo.search.yql.yqlplusParser.Orderby_fieldContext; import com.yahoo.search.yql.yqlplusParser.Output_specContext; -import com.yahoo.search.yql.yqlplusParser.Paged_clauseContext; -import com.yahoo.search.yql.yqlplusParser.ParamsContext; import com.yahoo.search.yql.yqlplusParser.Pipeline_stepContext; -import com.yahoo.search.yql.yqlplusParser.Procedure_argumentContext; -import com.yahoo.search.yql.yqlplusParser.Program_arglistContext; import com.yahoo.search.yql.yqlplusParser.Project_specContext; import com.yahoo.search.yql.yqlplusParser.ProgramContext; import com.yahoo.search.yql.yqlplusParser.PropertyNameAndValueContext; import com.yahoo.search.yql.yqlplusParser.Query_statementContext; import com.yahoo.search.yql.yqlplusParser.RelationalExpressionContext; import com.yahoo.search.yql.yqlplusParser.RelationalOpContext; -import com.yahoo.search.yql.yqlplusParser.Returning_specContext; import com.yahoo.search.yql.yqlplusParser.Scalar_literalContext; -import com.yahoo.search.yql.yqlplusParser.Select_source_joinContext; import com.yahoo.search.yql.yqlplusParser.Select_source_multiContext; import com.yahoo.search.yql.yqlplusParser.Select_statementContext; -import com.yahoo.search.yql.yqlplusParser.Selectvar_statementContext; import com.yahoo.search.yql.yqlplusParser.Sequence_sourceContext; import com.yahoo.search.yql.yqlplusParser.Source_listContext; import com.yahoo.search.yql.yqlplusParser.Source_specContext; import com.yahoo.search.yql.yqlplusParser.Source_statementContext; import com.yahoo.search.yql.yqlplusParser.StatementContext; import com.yahoo.search.yql.yqlplusParser.TimeoutContext; -import com.yahoo.search.yql.yqlplusParser.TypenameContext; import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext; -import com.yahoo.search.yql.yqlplusParser.Update_statementContext; -import com.yahoo.search.yql.yqlplusParser.Update_valuesContext; -import com.yahoo.search.yql.yqlplusParser.ViewContext; import com.yahoo.search.yql.yqlplusParser.WhereContext; import org.antlr.v4.runtime.BaseErrorListener; @@ -126,7 +96,6 @@ final class ProgramParser { return prepareParser(file.getAbsoluteFile().toString(), new CaseInsensitiveFileStream(file.getAbsolutePath())); } - private yqlplusParser prepareParser(String programName, CharStream input) { yqlplusLexer lexer = new yqlplusLexer(input); lexer.removeErrorListeners(); @@ -168,41 +137,18 @@ final class ProgramParser { try { return parser.program(); } catch (RecognitionException e) { - //Retry parsing using full LL mode + // Retry parsing using full LL mode parser.reset(); parser.getInterpreter().setPredictionMode(PredictionMode.LL); return parser.program(); } } - public OperatorNode<StatementOperator> parse(String programName, InputStream program) throws IOException, RecognitionException { - yqlplusParser parser = prepareParser(programName, program); - return convertProgram(parseProgram(parser), parser, programName); - } - public OperatorNode<StatementOperator> parse(String programName, String program) throws IOException, RecognitionException { yqlplusParser parser = prepareParser(programName, program); return convertProgram(parseProgram(parser), parser, programName); } - public OperatorNode<StatementOperator> parse(File input) throws IOException, RecognitionException { - yqlplusParser parser = prepareParser(input); - return convertProgram(parseProgram(parser), parser, input.getAbsoluteFile().toString()); - } - - public OperatorNode<ExpressionOperator> parseExpression(String input) throws IOException, RecognitionException { - return convertExpr(prepareParser("<expression>", input).expression(false).getRuleContext(), new Scope()); - } - - public OperatorNode<ExpressionOperator> parseExpression(String input, Set<String> visibleAliases) throws IOException, RecognitionException { - Scope scope = new Scope(); - final Location loc = new Location("<expression>", -1, -1); - for (String alias : visibleAliases) { - scope.defineDataSource(loc, alias); - } - return convertExpr(prepareParser("<expression>", input).expression(false).getRuleContext(), scope); - } - private Location toLocation(Scope scope, ParseTree node) { Token start; if (node instanceof ParserRuleContext) { @@ -212,8 +158,7 @@ final class ProgramParser { } else { throw new ProgramCompileException("Location is not available for type " + node.getClass()); } - Location location = new Location(scope != null? scope.programName: "<string>", start.getLine(), start.getCharPositionInLine()); - return location; + return new Location(scope != null? scope.programName: "<string>", start.getLine(), start.getCharPositionInLine()); } private List<String> readName(Namespaced_nameContext node) { @@ -230,14 +175,6 @@ final class ProgramParser { private final List<String> binding; - Binding(String moduleName, String exportName) { - this.binding = ImmutableList.of(moduleName, exportName); - } - - Binding(String moduleName) { - this.binding = ImmutableList.of(moduleName); - } - Binding(List<String> binding) { this.binding = binding; } @@ -263,13 +200,6 @@ final class ProgramParser { final yqlplusParser parser; final String programName; - Scope() { - this.parser = null; - this.programName = null; - this.root = this; - this.parent = null; - } - Scope(yqlplusParser parser, String programName) { this.parser = parser; this.programName = programName; @@ -288,15 +218,10 @@ final class ProgramParser { return parser; } - public String getProgramName() { - return programName; - } - public Set<String> getCursors() { return cursors; } - boolean isBound(String name) { // bindings live only in the 'root' node return root.bindings.containsKey(name); @@ -329,13 +254,6 @@ final class ProgramParser { root.bindings.put(symbolName, new Binding(binding)); } - public void bindModuleSymbol(Location loc, List<String> moduleName, String exportName, String symbolName) { - ImmutableList.Builder<String> builder = ImmutableList.builder(); - builder.addAll(moduleName); - builder.add(exportName); - bindModule(loc, builder.build(), symbolName); - } - public void defineDataSource(Location loc, String name) { if (isCursor(name)) { throw new ProgramCompileException(loc, "Alias '%s' is already used.", name); @@ -376,13 +294,12 @@ final class ProgramParser { } } - private OperatorNode<SequenceOperator> convertSelectOrInsertOrUpdateOrDelete(ParseTree node, Scope scopeParent) { + private OperatorNode<SequenceOperator> convertSelect(ParseTree node, Scope scopeParent) { - Preconditions.checkArgument(node instanceof Select_statementContext || node instanceof Insert_statementContext || - node instanceof Update_statementContext || node instanceof Delete_statementContext); + Preconditions.checkArgument(node instanceof Select_statementContext); - // SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? fallback? - // select is the only place to define where/orderby/limit/offset and joins + // SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? + // select is the only place to define where/orderby/limit/offset Scope scope = scopeParent.child(); ProjectionBuilder proj = null; OperatorNode<SequenceOperator> source = null; @@ -391,29 +308,18 @@ final class ProgramParser { OperatorNode<ExpressionOperator> offset = null; OperatorNode<ExpressionOperator> limit = null; OperatorNode<ExpressionOperator> timeout = null; - OperatorNode<SequenceOperator> fallback = null; - OperatorNode<SequenceOperator> insertValues = null; - OperatorNode<ExpressionOperator> updateValues = null; - ParseTree sourceNode; - - if (node instanceof Select_statementContext ) { - sourceNode = node.getChild(2) != null ? node.getChild(2).getChild(0):null; - } else { - sourceNode = node.getChild(1); - } + ParseTree sourceNode = node.getChild(2) != null ? node.getChild(2).getChild(0):null; if (sourceNode != null) { switch (getParseTreeIndex(sourceNode)) { // ALL_SOURCE and MULTI_SOURCE are how FROM SOURCES // *|source_name,... are parsed - // They can't be used directly with the JOIN syntax at this time - case yqlplusParser.RULE_select_source_all: { + case yqlplusParser.RULE_select_source_all: Location location = toLocation(scope, sourceNode.getChild(2)); source = OperatorNode.create(location, SequenceOperator.ALL); source.putAnnotation("alias", "row"); scope.defineDataSource(location, "row"); - } break; case yqlplusParser.RULE_select_source_multi: Source_listContext multiSourceContext = ((Select_source_multiContext) sourceNode).source_list(); @@ -421,22 +327,8 @@ final class ProgramParser { source.putAnnotation("alias", "row"); scope.defineDataSource(toLocation(scope, multiSourceContext), "row"); break; - case yqlplusParser.RULE_select_source_join: + case yqlplusParser.RULE_select_source_from: source = convertSource((ParserRuleContext) sourceNode.getChild(1), scope); - List<Join_exprContext> joinContexts = ((Select_source_joinContext)sourceNode).join_expr(); - for (Join_exprContext joinContext:joinContexts) { - source = convertJoin(joinContext, source, scope); - } - break; - case yqlplusParser.RULE_insert_source: - Insert_sourceContext insertSourceContext = (Insert_sourceContext) sourceNode; - source = convertSource((ParserRuleContext)insertSourceContext.getChild(1), scope); - break; - case yqlplusParser.RULE_delete_source: - source = convertSource((ParserRuleContext)sourceNode.getChild(1), scope); - break; - case yqlplusParser.RULE_update_source: - source = convertSource((ParserRuleContext)sourceNode.getChild(0), scope); break; } } else { @@ -451,9 +343,6 @@ final class ProgramParser { proj = readProjection(((Project_specContext) child.getChild(0)).field_def(), scope); } break; - case yqlplusParser.RULE_returning_spec: - proj = readProjection(((Returning_specContext) child).select_field_spec().project_spec().field_def(), scope); - break; case yqlplusParser.RULE_where: filter = convertExpr(((WhereContext) child).expression(), scope); break; @@ -475,23 +364,6 @@ final class ProgramParser { case yqlplusParser.RULE_timeout: timeout = convertExpr(((TimeoutContext) child).fixed_or_parameter(), scope); break; - case yqlplusParser.RULE_fallback: - fallback = convertQuery(((FallbackContext) child).select_statement(), scope); - break; - case yqlplusParser.RULE_insert_values: - if (child.getChild(0) instanceof yqlplusParser.Query_statementContext) { - insertValues = convertQuery(child.getChild(0).getChild(0), scope); - } else { - insertValues = readBatchValues(((Insert_valuesContext) child).field_names_spec(), ((Insert_valuesContext)child).field_values_group_spec(), scope); - } - break; - case yqlplusParser.RULE_update_values: - if (getParseTreeIndex(child.getChild(0)) == yqlplusParser.RULE_field_def) { - updateValues = readValues(((Update_valuesContext)child).field_def(), scope); - } else { - updateValues = readValues((Field_names_specContext)child.getChild(0), (Field_values_specContext)child.getChild(2), scope); - } - break; } } // now assemble the logical plan @@ -500,26 +372,6 @@ final class ProgramParser { if (filter != null) { result = OperatorNode.create(SequenceOperator.FILTER, result, filter); } - // insert values - if (insertValues != null) { - result = OperatorNode.create(SequenceOperator.INSERT, result, insertValues); - } - // update - if (updateValues != null) { - if (filter != null) { - result = OperatorNode.create(SequenceOperator.UPDATE, source, updateValues, filter); - } else { - result = OperatorNode.create(SequenceOperator.UPDATE_ALL, source, updateValues); - } - } - // delete - if (getParseTreeIndex(node) == yqlplusParser.RULE_delete_statement) { - if (filter != null) { - result = OperatorNode.create(SequenceOperator.DELETE, source, filter); - } else { - result = OperatorNode.create(SequenceOperator.DELETE_ALL, source); - } - } // then sort (or project and sort) boolean projectBeforeSort = false; if (orderby != null) { @@ -558,30 +410,9 @@ final class ProgramParser { if (timeout != null) { result = OperatorNode.create(SequenceOperator.TIMEOUT, result, timeout); } - // if there's a fallback, emit a fallback node - if (fallback != null) { - result = OperatorNode.create(SequenceOperator.FALLBACK, result, fallback); - } return result; } - private OperatorNode<ExpressionOperator> readValues(List<Field_defContext> fieldDefs, Scope scope) { - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - int numPairs = fieldDefs.size(); - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int j = 0; j < numPairs; j++) { - ParseTree startNode = fieldDefs.get(j); - while(startNode.getChildCount() < 3) { - startNode = startNode.getChild(0); - } - fieldNames.add((String) convertExpr(startNode.getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(startNode.getChild(2), scope)); - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - private OperatorNode<SequenceOperator> readMultiSource(Scope scope, Source_listContext multiSource) { List<List<String>> sourceNameList = Lists.newArrayList(); List<Namespaced_nameContext> nameSpaces = multiSource.namespaced_name(); @@ -591,9 +422,7 @@ final class ProgramParser { } return OperatorNode.create(toLocation(scope, multiSource), SequenceOperator.MULTISOURCE, sourceNameList); } -// pipeline_step -// : namespaced_name arguments[false]? -// ; + private OperatorNode<SequenceOperator> convertPipe(Query_statementContext queryStatementContext, List<Pipeline_stepContext> nodes, Scope scope) { OperatorNode<SequenceOperator> result = convertQuery(queryStatementContext.getChild(0), scope.getRoot()); for (Pipeline_stepContext step:nodes) { @@ -603,7 +432,7 @@ final class ProgramParser { } else { List<String> name = readName(step.namespaced_name()); List<OperatorNode<ExpressionOperator>> args = ImmutableList.of(); - //LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN + // LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN if (step.getChildCount() > 1) { ArgumentsContext arguments = step.arguments(); if (arguments.getChildCount() > 2) { @@ -621,56 +450,25 @@ final class ProgramParser { return result; } - private OperatorNode<SequenceOperator> convertMerge(List<Merge_componentContext> mergeComponentList, Scope scope) { - Preconditions.checkArgument(mergeComponentList != null); - List<OperatorNode<SequenceOperator>> sources = Lists.newArrayListWithExpectedSize(mergeComponentList.size()); - for (Merge_componentContext mergeComponent:mergeComponentList) { - Select_statementContext selectContext = mergeComponent.select_statement(); - Source_statementContext sourceContext = mergeComponent.source_statement(); - if (selectContext != null) { - sources.add(convertQuery(selectContext, scope.getRoot())); - } else { - sources.add(convertQuery(sourceContext, scope.getRoot())); - } - } - return OperatorNode.create(SequenceOperator.MERGE, sources); - } - private OperatorNode<SequenceOperator> convertQuery(ParseTree node, Scope scope) { - if (node instanceof Select_statementContext - || node instanceof Insert_statementContext - || node instanceof Update_statementContext - || node instanceof Delete_statementContext) { - return convertSelectOrInsertOrUpdateOrDelete(node, scope.getRoot()); + if (node instanceof Select_statementContext) { + return convertSelect(node, scope.getRoot()); } else if (node instanceof Source_statementContext) { // for pipe Source_statementContext sourceStatementContext = (Source_statementContext)node; return convertPipe(sourceStatementContext.query_statement(), sourceStatementContext.pipeline_step(), scope); - } else if (node instanceof Merge_statementContext) { - return convertMerge(((Merge_statementContext)node).merge_component(), scope); } else { throw new IllegalArgumentException("Unexpected argument type to convertQueryStatement: " + node.toStringTree()); } } - private OperatorNode<SequenceOperator> convertJoin(Join_exprContext node, OperatorNode<SequenceOperator> left, Scope scope) { - Source_specContext sourceSpec = node.source_spec(); - OperatorNode<SequenceOperator> right = convertSource(sourceSpec, scope); - JoinExpressionContext joinContext = node.joinExpression(); - OperatorNode<ExpressionOperator> joinExpression = readBinOp(ExpressionOperator.valueOf("EQ"), joinContext.getChild(0), joinContext.getChild(2), scope); - if (joinExpression.getOperator() != ExpressionOperator.EQ) { - throw new ProgramCompileException(joinExpression.getLocation(), "Unexpected join expression type: %s (expected EQ)", joinExpression.getOperator()); - } - return OperatorNode.create(toLocation(scope, sourceSpec), node.join_spec().LEFT() != null ? SequenceOperator.LEFT_JOIN : SequenceOperator.JOIN, left, right, joinExpression); - } - private String assignAlias(String alias, ParserRuleContext node, Scope scope) { if (alias == null) { alias = "source"; } - if (node != null && node instanceof yqlplusParser.Alias_defContext) { - //alias_def : (AS? ID); + if (node instanceof yqlplusParser.Alias_defContext) { + // alias_def : (AS? ID); ParseTree idChild = node; if (node.getChildCount() > 1) { idChild = node.getChild(1); @@ -690,20 +488,14 @@ final class ProgramParser { scope.defineDataSource(null, candidate); return alias; } - } - - private OperatorNode<SequenceOperator> convertSource(ParserRuleContext sourceSpecNode, Scope scope) { + } + private OperatorNode<SequenceOperator> convertSource(ParserRuleContext sourceSpecNode, Scope scope) { // DataSources String alias; OperatorNode<SequenceOperator> result; ParserRuleContext dataSourceNode = sourceSpecNode; ParserRuleContext aliasContext = null; - //data_source - //: call_source - //| LPAREN source_statement RPAREN - //| sequence_source - //; if (sourceSpecNode instanceof Source_specContext) { dataSourceNode = (ParserRuleContext)sourceSpecNode.getChild(0); if (sourceSpecNode.getChildCount() == 2) { @@ -717,7 +509,6 @@ final class ProgramParser { } } switch (getParseTreeIndex(dataSourceNode)) { - case yqlplusParser.RULE_write_data_source: case yqlplusParser.RULE_call_source: { List<String> names = readName(dataSourceNode.getChild(Namespaced_nameContext.class, 0)); alias = assignAlias(names.get(names.size() - 1), aliasContext, scope); @@ -763,60 +554,9 @@ final class ProgramParser { return result; } - private OperatorNode<TypeOperator> decodeType(Scope scope, TypenameContext type) { - - TypeOperator op; - ParseTree typeNode = type.getChild(0); - switch (getParseTreeIndex(typeNode)) { - case yqlplusParser.TYPE_BOOLEAN: - op = TypeOperator.BOOLEAN; - break; - case yqlplusParser.TYPE_BYTE: - op = TypeOperator.BYTE; - break; - case yqlplusParser.TYPE_DOUBLE: - op = TypeOperator.DOUBLE; - break; - case yqlplusParser.TYPE_INT16: - op = TypeOperator.INT16; - break; - case yqlplusParser.TYPE_INT32: - op = TypeOperator.INT32; - break; - case yqlplusParser.TYPE_INT64: - op = TypeOperator.INT64; - break; - case yqlplusParser.TYPE_STRING: - op = TypeOperator.STRING; - break; - case yqlplusParser.TYPE_TIMESTAMP: - op = TypeOperator.TIMESTAMP; - break; - case yqlplusParser.RULE_arrayType: - return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.ARRAY, decodeType(scope, ((ArrayTypeContext)typeNode).getChild(TypenameContext.class, 0))); - case yqlplusParser.RULE_mapType: - return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.MAP, decodeType(scope, ((MapTypeContext)typeNode).getChild(TypenameContext.class, 0))); - default: - throw new ProgramCompileException("Unknown type " + typeNode.getText()); - } - return OperatorNode.create(toLocation(scope, typeNode), op); - } - - private List<String> createBindingName(ParseTree node) { - if (node instanceof ModuleNameContext) { - if (((ModuleNameContext)node).namespaced_name() != null) { - return readName(((ModuleNameContext)node).namespaced_name()); - } else if (((ModuleNameContext)node).literalString() != null) { - return ImmutableList.of(((ModuleNameContext)node).literalString().STRING().getText()); - } - } else if (node instanceof ModuleIdContext) { - return ImmutableList.of(node.getText()); - } - throw new ProgramCompileException("Wrong context"); - } - - private OperatorNode<StatementOperator> convertProgram( - ParserRuleContext program, yqlplusParser parser, String programName) { + private OperatorNode<StatementOperator> convertProgram(ParserRuleContext program, + yqlplusParser parser, + String programName) { Scope scope = new Scope(parser, programName); List<OperatorNode<StatementOperator>> stmts = Lists.newArrayList(); int output = 0; @@ -825,148 +565,37 @@ final class ProgramParser { continue; } ParserRuleContext ruleContext = (ParserRuleContext) node; - switch (ruleContext.getRuleIndex()) { - case yqlplusParser.RULE_params: { - // ^(ARGUMENT ident typeref expression?) - ParamsContext paramsContext = (ParamsContext) ruleContext; - Program_arglistContext program_arglistContext = paramsContext.program_arglist(); - if (program_arglistContext != null) { - List<Procedure_argumentContext> argList = program_arglistContext.procedure_argument(); - for (Procedure_argumentContext procedureArgumentContext : argList) { - String name = procedureArgumentContext.ident().getText(); - OperatorNode<TypeOperator> type = decodeType(scope, procedureArgumentContext.getChild(TypenameContext.class, 0)); - OperatorNode<ExpressionOperator> defaultValue = OperatorNode.create(ExpressionOperator.NULL); - if (procedureArgumentContext.expression() != null) { - defaultValue = convertExpr(procedureArgumentContext.expression(), scope); - } - scope.defineVariable(toLocation(scope, procedureArgumentContext), name); - stmts.add(OperatorNode.create(StatementOperator.ARGUMENT, name, type, defaultValue)); - } - } - break; - } - case yqlplusParser.RULE_import_statement: { - Import_statementContext importContext = (Import_statementContext) ruleContext; - if (null == importContext.import_list()) { - List<String> name = createBindingName(node.getChild(1)); - String target; - Location location = toLocation(scope, node.getChild(1)); - if (node.getChildCount() == 2) { - target = name.get(0); - } else if (node.getChildCount() == 4) { - target = node.getChild(3).getText(); - } else { - throw new ProgramCompileException("Unknown node count for IMPORT: " + node.toStringTree()); - } - scope.bindModule(location, name, target); - } else { - // | FROM moduleName IMPORT import_list -> ^(IMPORT_FROM - // moduleName import_list+) - Import_listContext importListContext = importContext.import_list(); - List<String> name = createBindingName(importContext.moduleName()); - Location location = toLocation(scope, importContext.moduleName()); - List<ModuleIdContext> moduleIds = importListContext.moduleId(); - List<String> symbols = Lists.newArrayListWithExpectedSize(moduleIds.size()); - for (ModuleIdContext cnode : moduleIds) { - symbols.add(cnode.ID().getText()); - } - for (String sym : symbols) { - scope.bindModuleSymbol(location, name, sym, sym); - } - } - break; - } + if (ruleContext.getRuleIndex() != yqlplusParser.RULE_statement) + throw new ProgramCompileException("Unknown program element: " + node.getText()); - // DDL - case yqlplusParser.RULE_ddl: - ruleContext = (ParserRuleContext)ruleContext.getChild(0); - break; - case yqlplusParser.RULE_view: { - // view and projection expansion now has to be done by the - // execution engine - // since views/projections, in order to be useful, have to - // support being used from outside the same program - ViewContext viewContext = (ViewContext) ruleContext; - Location loc = toLocation(scope, viewContext); - scope.getRoot().defineView(loc, viewContext.ID().getText()); - stmts.add(OperatorNode.create(loc, StatementOperator.DEFINE_VIEW, viewContext.ID().getText(), convertQuery(viewContext.source_statement(), scope.getRoot()))); - break; + // ^(STATEMENT_QUERY source_statement paged_clause? output_spec?) + StatementContext statementContext = (StatementContext) ruleContext; + Source_statementContext source_statement = statementContext.output_statement().source_statement(); + OperatorNode<SequenceOperator> query; + if (source_statement.getChildCount() == 1) { + query = convertQuery( source_statement.query_statement().getChild(0), scope); + } else { + query = convertQuery(source_statement, scope); } - case yqlplusParser.RULE_statement: { - // ^(STATEMENT_QUERY source_statement paged_clause? - // output_spec?) - StatementContext statementContext = (StatementContext) ruleContext; - switch (getParseTreeIndex(ruleContext.getChild(0))) { - case yqlplusParser.RULE_selectvar_statement: { - // ^(STATEMENT_SELECTVAR ident source_statement) - Selectvar_statementContext selectVarContext = (Selectvar_statementContext) ruleContext.getChild(0); - String variable = selectVarContext.ident().getText(); - OperatorNode<SequenceOperator> query = convertQuery(selectVarContext.source_statement(), scope); - Location location = toLocation(scope, selectVarContext.ident()); - scope.defineVariable(location, variable); - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); - break; - } - case yqlplusParser.RULE_next_statement: { - // NEXT^ literalString OUTPUT! AS! ident - Next_statementContext nextStateContext = (Next_statementContext) ruleContext.getChild(0); - String continuationValue = StringUnescaper.unquote(nextStateContext.literalString().getText()); - String variable = nextStateContext.ident().getText(); - Location location = toLocation(scope, node); - OperatorNode<SequenceOperator> next = OperatorNode.create(location, SequenceOperator.NEXT, continuationValue); - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, next, variable)); - stmts.add(OperatorNode.create(location, StatementOperator.OUTPUT, variable)); - scope.defineVariable(location, variable); - break; + String variable = "result" + (++output); + boolean isCountVariable = false; + ParseTree outputStatement = node.getChild(0); + Location location = toLocation(scope, outputStatement); + for (int i = 1; i < outputStatement.getChildCount(); ++i) { + ParseTree child = outputStatement.getChild(i); + if ( getParseTreeIndex(child) != yqlplusParser.RULE_output_spec) + throw new ProgramCompileException( "Unknown statement attribute: " + child.toStringTree()); + + Output_specContext outputSpecContext = (Output_specContext) child; + variable = outputSpecContext.ident().getText(); + if (outputSpecContext.COUNT() != null) { + isCountVariable = true; } - case yqlplusParser.RULE_output_statement: - Source_statementContext source_statement = statementContext.output_statement().source_statement(); - OperatorNode<SequenceOperator> query; - if (source_statement.getChildCount() == 1) { - query = convertQuery( source_statement.query_statement().getChild(0), scope); - } else { - query = convertQuery(source_statement, scope); - } - String variable = "result" + (++output); - boolean isCountVariable = false; - OperatorNode<ExpressionOperator> pageSize = null; - ParseTree outputStatement = node.getChild(0); - Location location = toLocation(scope, outputStatement); - for (int i = 1; i < outputStatement.getChildCount(); ++i) { - ParseTree child = outputStatement.getChild(i); - switch (getParseTreeIndex(child)) { - case yqlplusParser.RULE_paged_clause: - Paged_clauseContext pagedContext = (Paged_clauseContext) child; - pageSize = convertExpr(pagedContext.fixed_or_parameter(), scope); - break; - case yqlplusParser.RULE_output_spec: - Output_specContext outputSpecContext = (Output_specContext) child; - variable = outputSpecContext.ident().getText(); - if (outputSpecContext.COUNT() != null) { - isCountVariable = true; - } - break; - default: - throw new ProgramCompileException( "Unknown statement attribute: " + child.toStringTree()); - } - } - scope.defineVariable(location, variable); - if (pageSize != null) { - query = OperatorNode.create(SequenceOperator.PAGE, query, pageSize); - } - stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); - stmts.add(OperatorNode.create(location, isCountVariable ? StatementOperator.COUNT:StatementOperator.OUTPUT, variable)); - } - break; - } - default: - throw new ProgramCompileException("Unknown program element: " + node.getText()); } + scope.defineVariable(location, variable); + stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable)); + stmts.add(OperatorNode.create(location, isCountVariable ? StatementOperator.COUNT:StatementOperator.OUTPUT, variable)); } - // traverse the tree, find all of the namespaced calls not covered by - // imports so we can - // define "implicit" import statements for them (to make engine - // implementation easier) return OperatorNode.create(StatementOperator.PROGRAM, stmts); } @@ -982,19 +611,19 @@ final class ProgramParser { private ProjectionBuilder readProjection(List<Field_defContext> fieldDefs, Scope scope) { if (null == fieldDefs) - throw new ProgramCompileException("Null fieldDefs"); + throw new ProgramCompileException("Null fieldDefs"); ProjectionBuilder proj = new ProjectionBuilder(); for (Field_defContext rulenode : fieldDefs) { // FIELD - // expression alias_def? - OperatorNode<ExpressionOperator> expr = convertExpr(rulenode.getChild(0), scope); + // expression alias_def? + OperatorNode<ExpressionOperator> expr = convertExpr(rulenode.getChild(0), scope); - String aliasName = null; - if (rulenode.getChildCount() > 1) { - // ^(ALIAS ID) - aliasName = rulenode.alias_def().ID().getText(); - } - proj.addField(aliasName, expr); + String aliasName = null; + if (rulenode.getChildCount() > 1) { + // ^(ALIAS ID) + aliasName = rulenode.alias_def().ID().getText(); + } + proj.addField(aliasName, expr); // no grammar for the other rule types at this time } return proj; @@ -1009,358 +638,345 @@ final class ProgramParser { } public OperatorNode<ExpressionOperator> convertExpr(ParseTree parseTree, Scope scope) { - switch (getParseTreeIndex(parseTree)) { - case yqlplusParser.RULE_vespa_grouping: { - ParseTree firstChild = parseTree.getChild(0); - if (getParseTreeIndex(firstChild) == yqlplusParser.RULE_annotation) { - ParseTree secondChild = parseTree.getChild(1); - OperatorNode<ExpressionOperator> annotation = convertExpr(((AnnotationContext) firstChild) - .constantMapExpression(), scope); - OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild), - ExpressionOperator.VESPA_GROUPING, secondChild.getText()); - List<String> names = annotation.getArgument(0); - List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); - for (int i = 0; i < names.size(); ++i) { - expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); - } - return expr; - } else { - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VESPA_GROUPING, - firstChild.getText()); - } - } - case yqlplusParser.RULE_nullOperator: - return OperatorNode.create(ExpressionOperator.NULL); - case yqlplusParser.RULE_argument: - return convertExpr(parseTree.getChild(0), scope); - case yqlplusParser.RULE_fixed_or_parameter: { - ParseTree firstChild = parseTree.getChild(0); - if (getParseTreeIndex(firstChild) == yqlplusParser.INT) { - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.LITERAL, Integer.valueOf(firstChild.getText())); - } else { - return convertExpr(firstChild, scope); - } - } - case yqlplusParser.RULE_constantMapExpression: { - List<ConstantPropertyNameAndValueContext> propertyList = ((ConstantMapExpressionContext) parseTree).constantPropertyNameAndValue(); - List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); - List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithExpectedSize(propertyList.size()); - for (ConstantPropertyNameAndValueContext child : propertyList) { - // : propertyName ':' expression[$expression::namespace] -> - // ^(PROPERTY propertyName expression) - names.add(StringUnescaper.unquote(child.getChild(0).getText())); - exprs.add(convertExpr(child.getChild(2), scope)); + switch (getParseTreeIndex(parseTree)) { + case yqlplusParser.RULE_vespa_grouping: { + ParseTree firstChild = parseTree.getChild(0); + if (getParseTreeIndex(firstChild) == yqlplusParser.RULE_annotation) { + ParseTree secondChild = parseTree.getChild(1); + OperatorNode<ExpressionOperator> annotation = convertExpr(((AnnotationContext) firstChild) + .constantMapExpression(), scope); + OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild), + ExpressionOperator.VESPA_GROUPING, secondChild.getText()); + List<String> names = annotation.getArgument(0); + List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); + for (int i = 0; i < names.size(); ++i) { + expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); + } + return expr; + } else { + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VESPA_GROUPING, + firstChild.getText()); + } } - return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); - } - case yqlplusParser.RULE_mapExpression: { - List<PropertyNameAndValueContext> propertyList = ((MapExpressionContext)parseTree).propertyNameAndValue(); - List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); - List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithCapacity(propertyList.size()); - for (PropertyNameAndValueContext child : propertyList) { - // : propertyName ':' expression[$expression::namespace] -> - // ^(PROPERTY propertyName expression) - names.add(StringUnescaper.unquote(child.getChild(0).getText())); - exprs.add(convertExpr(child.getChild(2), scope)); - } - return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); - } - case yqlplusParser.RULE_constantArray: { - List<ConstantExpressionContext> expressionList = ((ConstantArrayContext)parseTree).constantExpression(); - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ConstantExpressionContext expr : expressionList) { - values.add(convertExpr(expr, scope)); + case yqlplusParser.RULE_nullOperator: + return OperatorNode.create(ExpressionOperator.NULL); + case yqlplusParser.RULE_argument: + return convertExpr(parseTree.getChild(0), scope); + case yqlplusParser.RULE_fixed_or_parameter: { + ParseTree firstChild = parseTree.getChild(0); + if (getParseTreeIndex(firstChild) == yqlplusParser.INT) { + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.LITERAL, Integer.valueOf(firstChild.getText())); + } else { + return convertExpr(firstChild, scope); + } + } + case yqlplusParser.RULE_constantMapExpression: { + List<ConstantPropertyNameAndValueContext> propertyList = ((ConstantMapExpressionContext) parseTree).constantPropertyNameAndValue(); + List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); + List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithExpectedSize(propertyList.size()); + for (ConstantPropertyNameAndValueContext child : propertyList) { + // : propertyName ':' expression[$expression::namespace] -> + // ^(PROPERTY propertyName expression) + names.add(StringUnescaper.unquote(child.getChild(0).getText())); + exprs.add(convertExpr(child.getChild(2), scope)); + } + return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); + } + case yqlplusParser.RULE_mapExpression: { + List<PropertyNameAndValueContext> propertyList = ((MapExpressionContext)parseTree).propertyNameAndValue(); + List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size()); + List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithCapacity(propertyList.size()); + for (PropertyNameAndValueContext child : propertyList) { + // : propertyName ':' expression[$expression::namespace] -> + // ^(PROPERTY propertyName expression) + names.add(StringUnescaper.unquote(child.getChild(0).getText())); + exprs.add(convertExpr(child.getChild(2), scope)); + } + return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs); + } + case yqlplusParser.RULE_constantArray: { + List<ConstantExpressionContext> expressionList = ((ConstantArrayContext)parseTree).constantExpression(); + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); + for (ConstantExpressionContext expr : expressionList) { + values.add(convertExpr(expr, scope)); + } + return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + } + case yqlplusParser.RULE_arrayLiteral: { + List<ExpressionContext> expressionList = ((ArrayLiteralContext) parseTree).expression(); + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); + for (ExpressionContext expr : expressionList) { + values.add(convertExpr(expr, scope)); + } + return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + } + // dereferencedExpression: primaryExpression(indexref[in_select]| propertyref)* + case yqlplusParser.RULE_dereferencedExpression: { + DereferencedExpressionContext dereferencedExpression = (DereferencedExpressionContext) parseTree; + Iterator<ParseTree> it = dereferencedExpression.children.iterator(); + OperatorNode<ExpressionOperator> result = convertExpr(it.next(), scope); + while (it.hasNext()) { + ParseTree defTree = it.next(); + if (getParseTreeIndex(defTree) == yqlplusParser.RULE_propertyref) { + // DOT nm=ID + result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.PROPREF, result, defTree.getChild(1).getText()); + } else { + // indexref + result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.INDEX, result, convertExpr(defTree.getChild(1), scope)); + } + } + return result; + } + case yqlplusParser.RULE_primaryExpression: { + // ^(CALL namespaced_name arguments) + ParseTree firstChild = parseTree.getChild(0); + switch (getParseTreeIndex(firstChild)) { + case yqlplusParser.RULE_fieldref: { + return convertExpr(firstChild, scope); + } + case yqlplusParser.RULE_callExpresion: { + List<ArgumentContext> args = ((ArgumentsContext) firstChild.getChild(1)).argument(); + List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(args.size()); + for (ArgumentContext argContext : args) { + arguments.add(convertExpr(argContext.expression(),scope)); + } + return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.CALL, scope.resolvePath(readName((Namespaced_nameContext) firstChild.getChild(0))), arguments); + } + case yqlplusParser.RULE_parameter: + // external variable reference + return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VARREF, firstChild.getChild(1).getText()); + case yqlplusParser.RULE_scalar_literal: + case yqlplusParser.RULE_arrayLiteral: + case yqlplusParser.RULE_mapExpression: + return convertExpr(firstChild, scope); + case yqlplusParser.LPAREN: + return convertExpr(parseTree.getChild(1), scope); + } + break; + } + case yqlplusParser.RULE_parameter: { + // external variable reference + ParserRuleContext parameterContext = (ParserRuleContext) parseTree; + IdentContext identContext = parameterContext.getRuleContext(IdentContext.class, 0); + return OperatorNode.create(toLocation(scope, identContext), ExpressionOperator.VARREF, identContext.getText()); + } + case yqlplusParser.RULE_annotateExpression: { + //annotation logicalORExpression + AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation(); + OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope); + OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope); + List<String> names = annotation.getArgument(0); + List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); + for (int i = 0; i < names.size(); ++i) { + expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); + } + return expr; + } + case yqlplusParser.RULE_expression: { + return convertExpr(parseTree.getChild(0), scope); + } + case yqlplusParser.RULE_logicalANDExpression: + LogicalANDExpressionContext andExpressionContext = (LogicalANDExpressionContext) parseTree; + return readConjOp(ExpressionOperator.AND, andExpressionContext.equalityExpression(), scope); + case yqlplusParser.RULE_logicalORExpression: { + int childCount = parseTree.getChildCount(); + LogicalORExpressionContext logicalORExpressionContext = (LogicalORExpressionContext) parseTree; + if (childCount > 1) { + return readConjOrOp(ExpressionOperator.OR, logicalORExpressionContext, scope); + } else { + List<EqualityExpressionContext> equalityExpressionList = ((LogicalANDExpressionContext) parseTree.getChild(0)).equalityExpression(); + if (equalityExpressionList.size() > 1) { + return readConjOp(ExpressionOperator.AND, equalityExpressionList, scope); + } else { + return convertExpr(equalityExpressionList.get(0), scope); + } + } + } + case yqlplusParser.RULE_equalityExpression: { + EqualityExpressionContext equalityExpression = (EqualityExpressionContext) parseTree; + RelationalExpressionContext relationalExpressionContext = equalityExpression.relationalExpression(0); + OperatorNode<ExpressionOperator> expr = convertExpr(relationalExpressionContext, scope); + InNotInTargetContext inNotInTarget = equalityExpression.inNotInTarget(); + int childCount = equalityExpression.getChildCount(); + if (childCount == 1) { + return expr; + } + if (inNotInTarget != null) { + Literal_listContext literalListContext = inNotInTarget.literal_list(); + boolean isIN = equalityExpression.IN() != null; + if (literalListContext == null) { + Select_statementContext selectStatementContext = inNotInTarget.select_statement(); + OperatorNode<SequenceOperator> query = convertQuery(selectStatementContext, scope); + return OperatorNode.create(expr.getLocation(),isIN ? ExpressionOperator.IN_QUERY: ExpressionOperator.NOT_IN_QUERY, expr, query); + } else { + // we need to identify the type of the target; if it's a + // scalar we need to wrap it in a CREATE_ARRAY + // if it's already a CREATE ARRAY then it's fine, otherwise + // we need to know the variable type + // return readBinOp(node.getType() == yqlplusParser.IN ? + // ExpressionOperator.IN : ExpressionOperator.NOT_IN, node, + // scope); + return readBinOp(isIN ? ExpressionOperator.IN: ExpressionOperator.NOT_IN, equalityExpression.getChild(0), literalListContext, scope); + } + + } else { + ParseTree firstChild = equalityExpression.getChild(1); + if (equalityExpression.getChildCount() == 2) { + switch (getParseTreeIndex(firstChild)) { + case yqlplusParser.IS_NULL: + return readUnOp(ExpressionOperator.IS_NULL, relationalExpressionContext, scope); + case yqlplusParser.IS_NOT_NULL: + return readUnOp(ExpressionOperator.IS_NOT_NULL, relationalExpressionContext, scope); + } + } else { + switch (getParseTreeIndex(firstChild.getChild(0))) { + case yqlplusParser.EQ: + return readBinOp(ExpressionOperator.EQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NEQ: + return readBinOp(ExpressionOperator.NEQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.LIKE: + return readBinOp(ExpressionOperator.LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NOTLIKE: + return readBinOp(ExpressionOperator.NOT_LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.MATCHES: + return readBinOp(ExpressionOperator.MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.NOTMATCHES: + return readBinOp(ExpressionOperator.NOT_MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + case yqlplusParser.CONTAINS: + return readBinOp(ExpressionOperator.CONTAINS, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); + } + } + + } + break; + } + case yqlplusParser.RULE_relationalExpression: { + RelationalExpressionContext relationalExpressionContext = (RelationalExpressionContext) parseTree; + RelationalOpContext opContext = relationalExpressionContext.relationalOp(); + if (opContext != null) { + switch (getParseTreeIndex(relationalExpressionContext.relationalOp().getChild(0))) { + case yqlplusParser.LT: + return readBinOp(ExpressionOperator.LT, parseTree, scope); + case yqlplusParser.LTEQ: + return readBinOp(ExpressionOperator.LTEQ, parseTree, scope); + case yqlplusParser.GT: + return readBinOp(ExpressionOperator.GT, parseTree, scope); + case yqlplusParser.GTEQ: + return readBinOp(ExpressionOperator.GTEQ, parseTree, scope); + } + } else { + return convertExpr(relationalExpressionContext.additiveExpression(0), scope); + } + } + break; + case yqlplusParser.RULE_additiveExpression: + case yqlplusParser.RULE_multiplicativeExpression: { + if (parseTree.getChildCount() > 1) { + String opStr = parseTree.getChild(1).getText(); + switch (opStr) { + case "+": + return readBinOp(ExpressionOperator.ADD, parseTree, scope); + case "-": + return readBinOp(ExpressionOperator.SUB, parseTree, scope); + case "/": + return readBinOp(ExpressionOperator.DIV, parseTree, scope); + case "*": + return readBinOp(ExpressionOperator.MULT, parseTree, scope); + case "%": + return readBinOp(ExpressionOperator.MOD, parseTree, scope); + default: + if (parseTree.getChild(0) instanceof UnaryExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.toStringTree()); + } + } + } else { + if (parseTree.getChild(0) instanceof UnaryExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else if (parseTree.getChild(0) instanceof MultiplicativeExpressionContext) { + return convertExpr(parseTree.getChild(0), scope); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.getText()); + } + } + } + case yqlplusParser.RULE_unaryExpression: { + if (1 == parseTree.getChildCount()) { + return convertExpr(parseTree.getChild(0), scope); + } else if (2 == parseTree.getChildCount()) { + if ("-".equals(parseTree.getChild(0).getText())) { + return readUnOp(ExpressionOperator.NEGATE, parseTree, scope); + } else if ("!".equals(parseTree.getChild(0).getText())) { + return readUnOp(ExpressionOperator.NOT, parseTree, scope); + } + throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown unary operator " + parseTree.getText()); + } else { + throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown child count " + parseTree.getChildCount() + " of " + parseTree.getText()); + } + } + case yqlplusParser.RULE_fieldref: { + // all in-scope data sources should be defined in scope + // the 'first' field in a namespaced reference must be: + // - a field name if (and only if) there is exactly one data source + // in scope OR + // - an alias name, which will be followed by a field name + // ^(FIELDREF<FieldReference>[$expression::namespace] + // namespaced_name) + List<String> path = readName((Namespaced_nameContext) parseTree.getChild(0)); + Location loc = toLocation(scope, parseTree.getChild(0)); + String alias = path.get(0); + OperatorNode<ExpressionOperator> result = null; + int start = 0; + if (scope.isCursor(alias)) { + if (path.size() > 1) { + result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(1)); + start = 2; + } else { + result = OperatorNode.create(loc, ExpressionOperator.READ_RECORD, alias); + start = 1; + } + } else if (scope.isBound(alias)) { + return OperatorNode.create(loc, ExpressionOperator.READ_MODULE, scope.getBinding(alias).toPathWith(path.subList(1, path.size()))); + } else if (scope.getCursors().size() == 1) { + alias = scope.getCursors().iterator().next(); + result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(0)); + start = 1; + } else { + // ah ha, we can't end up with a 'loose' UDF call because it + // won't be a module or known alias + // so we need not support implicit imports for constants used in + // UDFs + throw new ProgramCompileException(loc, "Unknown field or alias '%s'", alias); + } + for (int idx = start; idx < path.size(); ++idx) { + result = OperatorNode.create(loc, ExpressionOperator.PROPREF, result, path.get(idx)); + } + return result; } - return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); + case yqlplusParser.RULE_scalar_literal: + return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.LITERAL, convertLiteral((Scalar_literalContext) parseTree)); + case yqlplusParser.RULE_constantExpression: + return convertExpr(parseTree.getChild(0), scope); + case yqlplusParser.RULE_literal_list: + if (getParseTreeIndex(parseTree.getChild(1)) == yqlplusParser.RULE_array_parameter) { + return convertExpr(parseTree.getChild(1), scope); + } else { + List<Literal_elementContext> elements = ((Literal_listContext) parseTree).literal_element(); + ParseTree firldElement = elements.get(0).getChild(0); + if (elements.size() == 1 && scope.getParser().isArrayParameter(firldElement)) { + return convertExpr(firldElement, scope); + } else { + List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(elements.size()); + for (Literal_elementContext child : elements) { + values.add(convertExpr(child.getChild(0), scope)); + } + return OperatorNode.create(toLocation(scope, elements.get(0)),ExpressionOperator.ARRAY, values); + } + } } - case yqlplusParser.RULE_arrayLiteral: { - List<ExpressionContext> expressionList = ((ArrayLiteralContext) parseTree).expression(); - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ExpressionContext expr : expressionList) { - values.add(convertExpr(expr, scope)); - } - return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values); - } - //dereferencedExpression: primaryExpression(indexref[in_select]| propertyref)* - case yqlplusParser.RULE_dereferencedExpression: { - DereferencedExpressionContext dereferencedExpression = (DereferencedExpressionContext) parseTree; - Iterator<ParseTree> it = dereferencedExpression.children.iterator(); - OperatorNode<ExpressionOperator> result = convertExpr(it.next(), scope); - while (it.hasNext()) { - ParseTree defTree = it.next(); - if (getParseTreeIndex(defTree) == yqlplusParser.RULE_propertyref) { - //DOT nm=ID - result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.PROPREF, result, defTree.getChild(1).getText()); - } else { - //indexref - result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.INDEX, result, convertExpr(defTree.getChild(1), scope)); - } - } - return result; - } - case yqlplusParser.RULE_primaryExpression: { - // ^(CALL namespaced_name arguments) - ParseTree firstChild = parseTree.getChild(0); - switch (getParseTreeIndex(firstChild)) { - case yqlplusParser.RULE_fieldref: { - return convertExpr(firstChild, scope); - } - case yqlplusParser.RULE_callExpresion: { - List<ArgumentContext> args = ((ArgumentsContext) firstChild.getChild(1)).argument(); - List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(args.size()); - for (ArgumentContext argContext : args) { - arguments.add(convertExpr(argContext.expression(),scope)); - } - return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.CALL, scope.resolvePath(readName((Namespaced_nameContext) firstChild.getChild(0))), arguments); - } - // TODO add processing this is not implemented in V3 - // case yqlplusParser.APPLY: - - case yqlplusParser.RULE_parameter: - // external variable reference - return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VARREF, firstChild.getChild(1).getText()); - case yqlplusParser.RULE_scalar_literal: - case yqlplusParser.RULE_arrayLiteral: - case yqlplusParser.RULE_mapExpression: - return convertExpr(firstChild, scope); - case yqlplusParser.LPAREN: - return convertExpr(parseTree.getChild(1), scope); - } - break; - } - - // TODO: Temporarily disable CAST - think through how types are named - // case yqlplusParser.CAST: { - // - // return new Cast() - // } - // return new CastExpression(payload); - case yqlplusParser.RULE_parameter: { - // external variable reference - ParserRuleContext parameterContext = (ParserRuleContext) parseTree; - IdentContext identContext = parameterContext.getRuleContext(IdentContext.class, 0); - return OperatorNode.create(toLocation(scope, identContext), ExpressionOperator.VARREF, identContext.getText()); - } - case yqlplusParser.RULE_annotateExpression: { - //annotation logicalORExpression - AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation(); - OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope); - OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope); - List<String> names = annotation.getArgument(0); - List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); - for (int i = 0; i < names.size(); ++i) { - expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); - } - return expr; - } - case yqlplusParser.RULE_expression: { - return convertExpr(parseTree.getChild(0), scope); - } - case yqlplusParser.RULE_logicalANDExpression: - LogicalANDExpressionContext andExpressionContext = (LogicalANDExpressionContext) parseTree; - return readConjOp(ExpressionOperator.AND, andExpressionContext.equalityExpression(), scope); - case yqlplusParser.RULE_logicalORExpression: { - int childCount = parseTree.getChildCount(); - LogicalORExpressionContext logicalORExpressionContext = (LogicalORExpressionContext) parseTree; - if (childCount > 1) { - return readConjOrOp(ExpressionOperator.OR, logicalORExpressionContext, scope); - } else { - List<EqualityExpressionContext> equalityExpressionList = ((LogicalANDExpressionContext) parseTree.getChild(0)).equalityExpression(); - if (equalityExpressionList.size() > 1) { - return readConjOp(ExpressionOperator.AND, equalityExpressionList, scope); - } else { - return convertExpr(equalityExpressionList.get(0), scope); - } - } - } - case yqlplusParser.RULE_equalityExpression: { - EqualityExpressionContext equalityExpression = (EqualityExpressionContext) parseTree; - RelationalExpressionContext relationalExpressionContext = equalityExpression.relationalExpression(0); - OperatorNode<ExpressionOperator> expr = convertExpr(relationalExpressionContext, scope); - InNotInTargetContext inNotInTarget = equalityExpression.inNotInTarget(); - int childCount = equalityExpression.getChildCount(); - if (childCount == 1) { - return expr; - } - if (inNotInTarget != null) { - Literal_listContext literalListContext = inNotInTarget.literal_list(); - boolean isIN = equalityExpression.IN() != null; - if (literalListContext == null) { - Select_statementContext selectStatementContext = inNotInTarget.select_statement(); - OperatorNode<SequenceOperator> query = convertQuery(selectStatementContext, scope); - return OperatorNode.create(expr.getLocation(),isIN ? ExpressionOperator.IN_QUERY: ExpressionOperator.NOT_IN_QUERY, expr, query); - } else { - // we need to identify the type of the target; if it's a - // scalar we need to wrap it in a CREATE_ARRAY - // if it's already a CREATE ARRAY then it's fine, otherwise - // we need to know the variable type - // return readBinOp(node.getType() == yqlplusParser.IN ? - // ExpressionOperator.IN : ExpressionOperator.NOT_IN, node, - // scope); - return readBinOp(isIN ? ExpressionOperator.IN: ExpressionOperator.NOT_IN, equalityExpression.getChild(0), literalListContext, scope); - } - - } else { - ParseTree firstChild = equalityExpression.getChild(1); - if (equalityExpression.getChildCount() == 2) { - switch (getParseTreeIndex(firstChild)) { - case yqlplusParser.IS_NULL: - return readUnOp(ExpressionOperator.IS_NULL, relationalExpressionContext, scope); - case yqlplusParser.IS_NOT_NULL: - return readUnOp(ExpressionOperator.IS_NOT_NULL, relationalExpressionContext, scope); - } - } else { - switch (getParseTreeIndex(firstChild.getChild(0))) { - case yqlplusParser.EQ: - return readBinOp(ExpressionOperator.EQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NEQ: - return readBinOp(ExpressionOperator.NEQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.LIKE: - return readBinOp(ExpressionOperator.LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NOTLIKE: - return readBinOp(ExpressionOperator.NOT_LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.MATCHES: - return readBinOp(ExpressionOperator.MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.NOTMATCHES: - return readBinOp(ExpressionOperator.NOT_MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - case yqlplusParser.CONTAINS: - return readBinOp(ExpressionOperator.CONTAINS, equalityExpression.getChild(0), equalityExpression.getChild(2), scope); - } - } - - } - break; - } - case yqlplusParser.RULE_relationalExpression: { - RelationalExpressionContext relationalExpressionContext = (RelationalExpressionContext) parseTree; - RelationalOpContext opContext = relationalExpressionContext.relationalOp(); - if (opContext != null) { - switch (getParseTreeIndex(relationalExpressionContext.relationalOp().getChild(0))) { - case yqlplusParser.LT: - return readBinOp(ExpressionOperator.LT, parseTree, scope); - case yqlplusParser.LTEQ: - return readBinOp(ExpressionOperator.LTEQ, parseTree, scope); - case yqlplusParser.GT: - return readBinOp(ExpressionOperator.GT, parseTree, scope); - case yqlplusParser.GTEQ: - return readBinOp(ExpressionOperator.GTEQ, parseTree, scope); - } - } else { - return convertExpr(relationalExpressionContext.additiveExpression(0), scope); - } - } - break; - case yqlplusParser.RULE_additiveExpression: - case yqlplusParser.RULE_multiplicativeExpression: { - if (parseTree.getChildCount() > 1) { - String opStr = parseTree.getChild(1).getText(); - switch (opStr) { - case "+": - return readBinOp(ExpressionOperator.ADD, parseTree, scope); - case "-": - return readBinOp(ExpressionOperator.SUB, parseTree, scope); - case "/": - return readBinOp(ExpressionOperator.DIV, parseTree, scope); - case "*": - return readBinOp(ExpressionOperator.MULT, parseTree, scope); - case "%": - return readBinOp(ExpressionOperator.MOD, parseTree, scope); - default: - if (parseTree.getChild(0) instanceof UnaryExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.toStringTree()); - } - } - } else { - if (parseTree.getChild(0) instanceof UnaryExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else if (parseTree.getChild(0) instanceof MultiplicativeExpressionContext) { - return convertExpr(parseTree.getChild(0), scope); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.getText()); - } - } - } - case yqlplusParser.RULE_unaryExpression: { - if (1 == parseTree.getChildCount()) { - return convertExpr(parseTree.getChild(0), scope); - } else if (2 == parseTree.getChildCount()) { - if ("-".equals(parseTree.getChild(0).getText())) { - return readUnOp(ExpressionOperator.NEGATE, parseTree, scope); - } else if ("!".equals(parseTree.getChild(0).getText())) { - return readUnOp(ExpressionOperator.NOT, parseTree, scope); - } - throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown unary operator " + parseTree.getText()); - } else { - throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown child count " + parseTree.getChildCount() + " of " + parseTree.getText()); - } - } - case yqlplusParser.RULE_fieldref: - case yqlplusParser.RULE_joinDereferencedExpression: { - // all in-scope data sources should be defined in scope - // the 'first' field in a namespaced reference must be: - // - a field name if (and only if) there is exactly one data source - // in scope OR - // - an alias name, which will be followed by a field name - // ^(FIELDREF<FieldReference>[$expression::namespace] - // namespaced_name) - List<String> path = readName((Namespaced_nameContext) parseTree.getChild(0)); - Location loc = toLocation(scope, parseTree.getChild(0)); - String alias = path.get(0); - OperatorNode<ExpressionOperator> result = null; - int start = 0; - if (scope.isCursor(alias)) { - if (path.size() > 1) { - result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(1)); - start = 2; - } else { - result = OperatorNode.create(loc, ExpressionOperator.READ_RECORD, alias); - start = 1; - } - } else if (scope.isBound(alias)) { - return OperatorNode.create(loc, ExpressionOperator.READ_MODULE, scope.getBinding(alias).toPathWith(path.subList(1, path.size()))); - } else if (scope.getCursors().size() == 1) { - alias = scope.getCursors().iterator().next(); - result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(0)); - start = 1; - } else { - // ah ha, we can't end up with a 'loose' UDF call because it - // won't be a module or known alias - // so we need not support implicit imports for constants used in - // UDFs - throw new ProgramCompileException(loc, "Unknown field or alias '%s'", alias); - } - for (int idx = start; idx < path.size(); ++idx) { - result = OperatorNode.create(loc, ExpressionOperator.PROPREF, result, path.get(idx)); - } - return result; - } - case yqlplusParser.RULE_scalar_literal: - return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.LITERAL, convertLiteral((Scalar_literalContext) parseTree)); - case yqlplusParser.RULE_insert_values: - return readValues((Insert_valuesContext) parseTree, scope); - case yqlplusParser.RULE_constantExpression: - return convertExpr(parseTree.getChild(0), scope); - case yqlplusParser.RULE_literal_list: - if (getParseTreeIndex(parseTree.getChild(1)) == yqlplusParser.RULE_array_parameter) { - return convertExpr(parseTree.getChild(1), scope); - } else { - List<Literal_elementContext> elements = ((Literal_listContext) parseTree).literal_element(); - ParseTree firldElement = elements.get(0).getChild(0); - if (elements.size() == 1 && scope.getParser().isArrayParameter(firldElement)) { - return convertExpr(firldElement, scope); - } else { - List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(elements.size()); - for (Literal_elementContext child : elements) { - values.add(convertExpr(child.getChild(0), scope)); - } - return OperatorNode.create(toLocation(scope, elements.get(0)),ExpressionOperator.ARRAY, values); - } - } - } - throw new ProgramCompileException(toLocation(scope, parseTree), - "Unknown expression type: " + parseTree.getText()); + throw new ProgramCompileException(toLocation(scope, parseTree), + "Unknown expression type: " + parseTree.getText()); } public Object convertLiteral(Scalar_literalContext literal) { @@ -1462,77 +1078,7 @@ final class ProgramParser { } } - private OperatorNode<ExpressionOperator> readValues(Field_names_specContext nameDefs, Field_values_specContext values, Scope scope) { - List<Field_defContext> fieldDefs = nameDefs.field_def(); - List<ExpressionContext> valueDefs = values.expression(); - assert fieldDefs.size() == valueDefs.size(); - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - int numPairs = fieldDefs.size(); - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int i = 0; i < numPairs; i++) { - fieldNames.add((String) convertExpr(fieldDefs.get(i).expression(), scope).getArgument(1)); - fieldValues.add(convertExpr(valueDefs.get(i), scope)); - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - - private OperatorNode<ExpressionOperator> readValues(ParserRuleContext node, Scope scope) { - List<String> fieldNames; - List<OperatorNode<ExpressionOperator>> fieldValues; - if (node.getRuleIndex() == yqlplusParser.RULE_field_def) { - Field_defContext fieldDefContext = (Field_defContext)node; - //TODO double check - fieldNames = Lists.newArrayListWithExpectedSize(node.getChildCount()); - fieldValues = Lists.newArrayListWithExpectedSize(node.getChildCount()); - for (int i = 0; i < node.getChildCount(); i++) { - fieldNames.add((String) convertExpr(node.getChild(i).getChild(0).getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(node.getChild(i).getChild(0).getChild(1), scope)); - } - } else { - assert node.getChildCount() % 2 == 0; - int numPairs = node.getChildCount() / 2; - fieldNames = Lists.newArrayListWithExpectedSize(numPairs); - fieldValues = Lists.newArrayListWithExpectedSize(numPairs); - for (int i = 0; i < numPairs; i++) { - fieldNames.add((String) convertExpr(node.getChild(i).getChild(0), scope).getArgument(1)); - fieldValues.add(convertExpr(node.getChild(numPairs + i), scope)); - } - } - return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues); - } - - /* - * Converts node list - * - * a_name, b_name, c_name, a_value_1, b_value_1, c_value_1, a_value_2, b_value_2, c_value2, a_value_3, b_value_3, c_value_3 - * - * into corresponding constant sequence: - * - * [ { a_name : a_value_1, b_name : b_value_1, c_name : c_value_1 }, ... ] - * - */ - private OperatorNode<SequenceOperator> readBatchValues(Field_names_specContext nameDefs, List<Field_values_group_specContext> valueGroups, Scope scope) { - List<Field_defContext> nameContexts = nameDefs.field_def(); - List<String> fieldNames = Lists.newArrayList(); - for (Field_defContext nameContext:nameContexts) { - fieldNames.add((String) convertExpr(nameContext.getChild(0), scope).getArgument(1)); - } - List<OperatorNode> records = Lists.newArrayList(); - for (Field_values_group_specContext valueGorup:valueGroups) { - List<ExpressionContext> expressionList = valueGorup.expression(); - List<OperatorNode<ExpressionOperator>> fieldValues = Lists.newArrayListWithExpectedSize(expressionList.size()); - for (ExpressionContext expressionContext:expressionList) { - fieldValues.add(convertExpr(expressionContext, scope)); - } - records.add(OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues)); - } - // Return constant sequence of records with the given name/values - return OperatorNode.create(SequenceOperator.EVALUATE, OperatorNode.create(ExpressionOperator.ARRAY, records)); - } - - /* + /** * Scans the given node for READ_FIELD expressions. * * TODO: Search recursively and consider additional operators @@ -1557,4 +1103,5 @@ final class ProgramParser { } return readFieldList; } + } |