diff options
98 files changed, 2076 insertions, 499 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index a0dba36fef5..2ea1ef186d2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -83,7 +83,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int metricsproxyNumThreads() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default int largeRankExpressionLimit() { return 8192; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.526") default int largeRankExpressionLimit() { return 8192; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { throw new UnsupportedOperationException("TODO specify default value"); } @@ -101,7 +101,6 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } @ModelFeatureFlag(owners = {"arnej"}) default boolean newLocationBrokerLogic() { return true; } - @ModelFeatureFlag(owners = {"bjorncs"}, removeAfter = "7.504") default int maxConnectionLifeInHosted() { return 45; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int distributorMergeBusyWait() { return 10; } @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean distributorEnhancedMaintenanceScheduling() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2ff9b889a9e..0007fa0d18a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -59,7 +59,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxConcurrentMergesPerNode = 16; private int maxMergeQueueSize = 1024; private boolean ignoreMergeQueueLimit = false; - private int largeRankExpressionLimit = 8192; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = Collections.emptyList(); private double resourceLimitDisk = 0.8; @@ -115,7 +114,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } - @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @@ -160,10 +158,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea containerShutdownTimeout = value; return this; } - public TestProperties largeRankExpressionLimit(int value) { - largeRankExpressionLimit = value; - return this; - } + public TestProperties setFeedConcurrency(double feedConcurrency) { this.feedConcurrency = feedConcurrency; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java index 3fdea71da2c..a1299c12307 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java @@ -10,9 +10,14 @@ import java.util.concurrent.ConcurrentHashMap; public class LargeRankExpressions { private final Map<String, RankExpressionBody> expressions = new ConcurrentHashMap<>(); private final FileRegistry fileRegistry; + private final int limit; public LargeRankExpressions(FileRegistry fileRegistry) { + this(fileRegistry, 8192); + } + public LargeRankExpressions(FileRegistry fileRegistry, int limit) { this.fileRegistry = fileRegistry; + this.limit = limit; } public void add(RankExpressionBody expression) { @@ -29,6 +34,7 @@ public class LargeRankExpressions { } } } + public int limit() { return limit; } /** Returns a read-only map of the ranking constants in this indexed by name */ public Map<String, RankExpressionBody> asMap() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index a33498a37ec..d7bcd295f09 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -140,7 +140,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final int numSearchPartitions; private final double termwiseLimit; private final double rankScoreDropLimit; - private final int largeRankExpressionLimit; /** * The rank type definitions used to derive settings for the native rank features @@ -176,7 +175,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer { keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); - largeRankExpressionLimit = deployProperties.featureFlags().largeRankExpressionLimit(); rankProperties = new ArrayList<>(compiled.getRankProperties()); Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); @@ -419,7 +417,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (ListIterator<Pair<String, String>> iter = properties.listIterator(); iter.hasNext();) { Pair<String, String> property = iter.next(); String expression = property.getSecond(); - if (expression.length() > largeRankExpressionLimit) { + if (expression.length() > largeRankExpressions.limit()) { String propertyName = property.getFirst(); String functionName = RankingExpression.extractScriptName(propertyName); if (functionName != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 8ac30f66ae7..302e8eff2d8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -47,7 +47,7 @@ public class ConfigserverCluster extends AbstractConfigProducer this.containerCluster = containerCluster; // If we are in a config server cluster the correct zone is propagated through cloud config options, - // not through config to deployment options (see StandaloneContainerApplication.scala), + // not through config to deployment options (see StandaloneContainerApplication.java), // so we need to propagate the zone options into the container from here Environment environment = options.environment().isPresent() ? Environment.from(options.environment().get()) : Environment.defaultEnvironment(); RegionName region = options.region().isPresent() ? RegionName.from(options.region().get()) : RegionName.defaultName(); @@ -83,6 +83,8 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.zookeeperClientPort().isPresent()) { builder.clientPort(options.zookeeperClientPort().get()); } + + builder.snapshotMethod(options.zooKeeperSnapshotMethod()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index fe3bd271f2f..c61c140c05b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -38,4 +38,5 @@ public interface CloudConfigOptions { Optional<String> loadBalancerAddress(); Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); + String zooKeeperSnapshotMethod(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index c6a10c5530b..fda3e6c16c3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -104,8 +104,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testRankExpression() throws IOException, ParseException { - assertCorrectDeriving("rankexpression", null, - new TestProperties().largeRankExpressionLimit(1024), new TestableDeployLogger()); + assertCorrectDeriving("rankexpression"); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index ebc8ceacc50..f4742be6b30 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -115,9 +115,9 @@ public class RankingExpressionsTestCase extends AbstractSchemaTestCase { @Test public void testLargeInheritedFunctions() throws IOException, ParseException { - ModelContext.Properties properties = new TestProperties().largeRankExpressionLimit(50); + ModelContext.Properties properties = new TestProperties(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry()); + LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry(), 50); QueryProfileRegistry queryProfiles = new QueryProfileRegistry(); ImportedMlModels models = new ImportedMlModels(); Schema schema = createSearch("src/test/examples/largerankingexpressions", properties, rankProfileRegistry); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 7f49efa8770..4ca85a19c35 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -47,16 +47,17 @@ public class ConfigserverClusterTest { } @Test - public void zookeeperConfig_only_config_servers_set() { + public void zookeeperConfig_only_config_servers_set_hosted() { TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Collections.emptyList()); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 0, 1, 2); assertEquals(1, config.myid()); + assertEquals("gz", config.snapshotMethod()); } @Test - public void zookeeperConfig_with_config_servers_and_zk_ids() { + public void zookeeperConfig_with_config_servers_and_zk_ids_hosted() { TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Arrays.asList(4, 2, 3)); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); @@ -72,6 +73,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); assertEquals(2, config.myid()); + assertEquals("", config.snapshotMethod()); } @Test(expected = IllegalArgumentException.class) @@ -150,7 +152,8 @@ public class ConfigserverClusterTest { .useVespaVersionInRequest(true) .hostedVespa(hostedVespa) .environment("test") - .region("bar"); + .region("bar") + .zooKeeperSnapshotMethod(hostedVespa ? "gz" : ""); Optional.of(configServerHostnames) .filter(hostnames -> !hostnames.isEmpty()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index 9352dac85a4..fc7f8674149 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; +import java.util.Objects; import java.util.Optional; /** @@ -17,6 +18,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); + private String zooKeeperSnapshotMethod = ""; @Override public Optional<Integer> rpcPort() { @@ -106,6 +108,9 @@ public class TestOptions implements CloudConfigOptions { return Optional.empty(); } + @Override + public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } + public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -130,4 +135,11 @@ public class TestOptions implements CloudConfigOptions { this.hostedVespa = Optional.of(hostedVespa); return this; } + + public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { + Objects.requireNonNull(snapshotMethod); + this.zooKeeperSnapshotMethod = snapshotMethod; + return this; + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java index 68690bd83cd..8d83ec4cc5f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container.search; import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleImporter; import com.yahoo.prelude.semantics.SemanticRulesConfig; @@ -39,7 +40,7 @@ public class SemanticRulesTest { } private static Map<String, RuleBase> toMap(SemanticRulesConfig config) throws ParseException, IOException { - RuleImporter ruleImporter = new RuleImporter(config); + RuleImporter ruleImporter = new RuleImporter(config, new SimpleLinguistics()); Map<String, RuleBase> ruleBaseMap = new HashMap<>(); for (SemanticRulesConfig.Rulebase ruleBaseConfig : config.rulebase()) { RuleBase ruleBase = ruleImporter.importConfig(ruleBaseConfig); diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index dca81d44adb..d80ccc4d042 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -54,3 +54,4 @@ tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIF jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks" dynamicReconfiguration bool default=false +snapshotMethod string default="" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index a77a8d1b5b8..7d4ac948194 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -184,7 +184,6 @@ public class ModelContextImpl implements ModelContext { private final int maxConcurrentMergesPerContentNode; private final int maxMergeQueueSize; private final boolean ignoreMergeQueueLimit; - private final int largeRankExpressionLimit; private final double resourceLimitDisk; private final double resourceLimitMemory; private final double minNodeRatioPerGroup; @@ -226,7 +225,6 @@ public class ModelContextImpl implements ModelContext { this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); - this.largeRankExpressionLimit = flagValue(source, appId, Flags.LARGE_RANK_EXPRESSION_LIMIT); this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); this.ignoreMergeQueueLimit = flagValue(source, appId, Flags.IGNORE_MERGE_QUEUE_LIMIT); @@ -273,7 +271,6 @@ public class ModelContextImpl implements ModelContext { @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return translateJvmOmitStackTraceInFastThrowIntToString(jvmOmitStackTraceInFastThrow, type); } - @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 70e7dbe19ec..b3c2fa2e300 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -6,7 +6,6 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.concurrent.DaemonThreadFactory; -import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.Lock; import com.yahoo.concurrent.Locks; import com.yahoo.concurrent.StripedExecutor; @@ -36,7 +35,6 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.state.ConnectionState; @@ -202,7 +200,7 @@ public class TenantRepository { this.tenantListener = tenantListener; this.zookeeperServerConfig = zookeeperServerConfig; // This we should control with a feature flag. - this.deployHelperExecutor = createModelBuilderExecutor(Flags.NUM_DEPLOY_HELPER_THREADS.bindTo(flagSource).value()); + this.deployHelperExecutor = createModelBuilderExecutor(); curator.framework().getConnectionStateListenable().addListener(this::stateChanged); @@ -220,14 +218,11 @@ public class TenantRepository { TimeUnit.SECONDS); } - private ExecutorService createModelBuilderExecutor(int numThreads) { + private ExecutorService createModelBuilderExecutor() { final long GB = 1024*1024*1024; - if (numThreads == 0) return new InThreadExecutorService(); - if (numThreads < 0) { - long maxHeap = Runtime.getRuntime().maxMemory(); - int maxThreadsToFitInMemory = (int)((maxHeap + (GB - 1))/(1*GB)); - numThreads = Math.min(Runtime.getRuntime().availableProcessors(), maxThreadsToFitInMemory); - } + long maxHeap = Runtime.getRuntime().maxMemory(); + int maxThreadsToFitInMemory = (int)((maxHeap + (GB - 1))/(1*GB)); + int numThreads = Math.min(Runtime.getRuntime().availableProcessors(), maxThreadsToFitInMemory); return Executors.newFixedThreadPool(numThreads, ThreadFactoryFactory.getDaemonThreadFactory("deploy-helper")); } diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index d5be3ab52f2..bb8317c298b 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -2667,6 +2667,18 @@ "public static final enum com.yahoo.metrics.simple.UntypedMetric$AssumedType COUNTER" ] }, + "com.yahoo.metrics.simple.UntypedMetric$Histogram": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public double getValueAtPercentile(double)", + "public void outputPercentileDistribution(java.io.PrintStream, int, java.lang.Double, boolean)" + ], + "fields": [] + }, "com.yahoo.metrics.simple.UntypedMetric": { "superClass": "java.lang.Object", "interfaces": [], @@ -2680,7 +2692,7 @@ "public double getMax()", "public double getMin()", "public double getSum()", - "public org.HdrHistogram.DoubleHistogram getHistogram()", + "public com.yahoo.metrics.simple.UntypedMetric$Histogram getHistogram()", "public java.lang.String toString()" ], "fields": [] diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java b/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java index ad549fb4d91..3d82e7853d9 100644 --- a/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java +++ b/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.metrics.simple; -import java.util.logging.Logger; - +import com.yahoo.api.annotations.Beta; import org.HdrHistogram.DoubleHistogram; +import java.io.PrintStream; import java.util.logging.Level; +import java.util.logging.Logger; /** * A gauge or a counter or... who knows? The class for storing a metric when the @@ -114,8 +115,9 @@ public class UntypedMetric { return metricSettings; } - public DoubleHistogram getHistogram() { - return histogram; + @Beta + public Histogram getHistogram() { + return histogram != null ? new Histogram(histogram) : null; } @Override @@ -139,4 +141,19 @@ public class UntypedMetric { return buf.toString(); } + @Beta + public static class Histogram { + private final DoubleHistogram hdrHistogram; + + private Histogram(DoubleHistogram hdrHistogram) { this.hdrHistogram = hdrHistogram; } + + public double getValueAtPercentile(double percentile) { return hdrHistogram.getValueAtPercentile(percentile); } + + public void outputPercentileDistribution(PrintStream printStream, int percentileTicksPerHalfDistance, + Double outputValueUnitScalingRatio, boolean useCsvFormat) { + hdrHistogram.outputPercentileDistribution( + printStream, percentileTicksPerHalfDistance, outputValueUnitScalingRatio, useCsvFormat); + } + } + } diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java index 5b5fb67f1b4..4cd0d820433 100644 --- a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java +++ b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java @@ -1,21 +1,30 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.metrics.simple.jdisc; -import java.io.PrintStream; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -import org.HdrHistogram.DoubleHistogram; - import com.yahoo.collections.Tuple2; -import com.yahoo.container.jdisc.state.*; +import com.yahoo.container.jdisc.state.CountMetric; +import com.yahoo.container.jdisc.state.GaugeMetric; +import com.yahoo.container.jdisc.state.MetricDimensions; +import com.yahoo.container.jdisc.state.MetricSet; +import com.yahoo.container.jdisc.state.MetricSnapshot; +import com.yahoo.container.jdisc.state.MetricValue; +import com.yahoo.container.jdisc.state.StateMetricContext; import com.yahoo.metrics.simple.Bucket; import com.yahoo.metrics.simple.Identifier; import com.yahoo.metrics.simple.Point; import com.yahoo.metrics.simple.UntypedMetric; +import com.yahoo.metrics.simple.UntypedMetric.Histogram; import com.yahoo.metrics.simple.Value; -import com.yahoo.text.JSON; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; /** * Convert simple metrics snapshots into jdisc state snapshots. @@ -74,7 +83,7 @@ class SnapshotConverter { } } - private static List<Tuple2<String, Double>> buildPercentileList(DoubleHistogram histogram) { + private static List<Tuple2<String, Double>> buildPercentileList(Histogram histogram) { List<Tuple2<String, Double>> prefixAndValues = new ArrayList<>(2); prefixAndValues.add(new Tuple2<>("95", histogram.getValueAtPercentile(95.0d))); prefixAndValues.add(new Tuple2<>("99", histogram.getValueAtPercentile(99.0d))); @@ -122,7 +131,7 @@ class SnapshotConverter { continue; } gotHistogram = true; - DoubleHistogram histogram = entry.getValue().getHistogram(); + Histogram histogram = entry.getValue().getHistogram(); Identifier id = entry.getKey(); String metricIdentifier = getIdentifierString(id); output.println("# start of metric " + metricIdentifier); diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 6201ec4bfd9..b320a1090ae 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5538,7 +5538,6 @@ "public void setLocale(java.lang.String, com.yahoo.search.query.Sorting$UcaSorter$Strength)", "public java.lang.String getLocale()", "public com.yahoo.search.query.Sorting$UcaSorter$Strength getStrength()", - "public com.ibm.icu.text.Collator getCollator()", "public java.lang.String getDecomposition()", "public java.lang.String toSerialForm()", "public int hashCode()", diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index 2b8515b6db8..8e137d99951 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -1,19 +1,34 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.StemMode; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; +import com.yahoo.prelude.semantics.rule.CompositeCondition; +import com.yahoo.prelude.semantics.rule.Condition; +import com.yahoo.prelude.semantics.rule.NamedCondition; +import com.yahoo.prelude.semantics.rule.ProductionRule; +import com.yahoo.prelude.semantics.rule.SuperCondition; import com.yahoo.search.Query; import com.yahoo.prelude.querytransform.PhraseMatcher; import com.yahoo.prelude.semantics.engine.RuleEngine; import com.yahoo.prelude.semantics.parser.ParseException; -import com.yahoo.prelude.semantics.rule.*; import com.yahoo.protect.Validator; import java.io.File; -import java.util.*; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; /** - * A set of semantic production rules and named conditions used to analyze - * and rewrite queries + * A set of semantic production rules and named conditions used to analyze and rewrite queries * * @author bratseth */ @@ -26,7 +41,7 @@ public class RuleBase { private String source; /** The name of the automata file used, or null if none */ - protected String automataFileName = null; + private String automataFileName = null; /** * True if this rule base is default. @@ -61,29 +76,26 @@ public class RuleBase { */ private boolean usesAutomata = false; - /** Should we allow stemmed matches? */ - private boolean stemming = true; - - /** Creates an empty rule base. TODO: Disallow */ - public RuleBase() { - } + private RuleBaseLinguistics linguistics; /** Creates an empty rule base */ - public RuleBase(String name) { - setName(name); + public RuleBase(String name, Linguistics linguistics) { + this.name = name; + this.linguistics = new RuleBaseLinguistics(StemMode.BEST, Language.ENGLISH, linguistics); } /** - * Creates a rule base from a file + * Creates a rule base from file * - * @param ruleFile the rule file to read. The name of the file (minus path) becomes the rule base name + * @param ruleFile the rule file to read. The name of the file (minus path) becomes the rule base name. * @param automataFile the automata file, or null to not use an automata * @throws java.io.IOException if there is a problem reading one of the files * @throws ParseException if the rule file can not be parsed correctly * @throws RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromFile(String ruleFile, String automataFile) throws java.io.IOException, ParseException { - return new RuleImporter().importFile(ruleFile, automataFile); + public static RuleBase createFromFile(String ruleFile, String automataFile, Linguistics linguistics) + throws java.io.IOException, ParseException { + return new RuleImporter(linguistics).importFile(ruleFile, automataFile); } /** @@ -96,18 +108,13 @@ public class RuleBase { * @throws com.yahoo.prelude.semantics.parser.ParseException if the rule file can not be parsed correctly * @throws com.yahoo.prelude.semantics.RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromString(String name, String ruleString, String automataFile) throws java.io.IOException, ParseException { - RuleBase base = new RuleImporter().importString(ruleString, automataFile, new RuleBase()); + public static RuleBase createFromString(String name, String ruleString, String automataFile, Linguistics linguistics) + throws java.io.IOException, ParseException { + RuleBase base = new RuleImporter(linguistics).importString(ruleString, automataFile); base.setName(name); return base; } - /** Set to true to enable stemmed matches. True by default */ - public void setStemming(boolean stemming) { this.stemming = stemming; } - - /** Returns whether stemmed matches are allowed. True by default */ - public boolean getStemming() { return stemming; } - /** * <p>Include another rule base into this. This <b>transfers ownership</b> * of the given rule base - it can not be subsequently used for any purpose @@ -171,7 +178,7 @@ public class RuleBase { resolveSuper(condition, superCondition); } - private void resolveSuper(Condition condition,Condition superCondition) { + private void resolveSuper(Condition condition, Condition superCondition) { if (condition instanceof SuperCondition) { ((SuperCondition)condition).setCondition(superCondition); } @@ -336,7 +343,7 @@ public class RuleBase { // TODO: Values are not added right now protected void annotatePhrase(PhraseMatcher.Phrase phrase,Query query,int traceLevel) { - for (StringTokenizer tokens = new StringTokenizer(phrase.getData(),"|",false) ; tokens.hasMoreTokens(); ) { + for (StringTokenizer tokens = new StringTokenizer(phrase.getData(), "|", false); tokens.hasMoreTokens(); ) { String token = tokens.nextToken(); int semicolonIndex = token.indexOf(";"); String annotation = token; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java index 45569050882..acbf9a7ffb6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java @@ -10,8 +10,9 @@ import java.util.Arrays; import java.util.List; import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; -import com.yahoo.prelude.semantics.parser.*; +import com.yahoo.language.Linguistics; +import com.yahoo.prelude.semantics.parser.ParseException; +import com.yahoo.prelude.semantics.parser.SemanticsParser; /** * Imports rule bases from various sources. @@ -24,51 +25,47 @@ import com.yahoo.prelude.semantics.parser.*; // rule bases included into others, while neither the rule base or the parser knows. public class RuleImporter { - /** - * If this is set, imported rule bases are looked up in this config - * otherwise, they are looked up as files - */ - private SemanticRulesConfig config; + /** If this is set, imported rule bases are looked up in this config otherwise, they are looked up as files. */ + private final SemanticRulesConfig config; - /** - * Ignore requests to read automata files. - * Useful to validate rule bases without having automatas present - */ - private boolean ignoreAutomatas; + /** Ignore requests to read automata files. Useful to validate rule bases without having automatas present. */ + private final boolean ignoreAutomatas; - /** - * Ignore requests to include files. - * Useful to validate rule bases one by one in config - */ - private boolean ignoreIncludes = false; + /** Ignore requests to include files. Useful to validate rule bases one by one in config. */ + private final boolean ignoreIncludes; + + private Linguistics linguistics; /** Create a rule importer which will read from file */ - public RuleImporter() { - this(null, false); + public RuleImporter(Linguistics linguistics) { + this(null, false, linguistics); } /** Create a rule importer which will read from a config object */ - public RuleImporter(SemanticRulesConfig config) { - this(config, false); + public RuleImporter(SemanticRulesConfig config, Linguistics linguistics) { + this(config, false, linguistics); } - public RuleImporter(boolean ignoreAutomatas) { - this(null, ignoreAutomatas); + public RuleImporter(boolean ignoreAutomatas, Linguistics linguistics) { + this(null, ignoreAutomatas, linguistics); } - public RuleImporter(boolean ignoreAutomatas, boolean ignoreIncludes) { - this(null, ignoreAutomatas, ignoreIncludes); + public RuleImporter(boolean ignoreAutomatas, boolean ignoreIncludes, Linguistics linguistics) { + this(null, ignoreAutomatas, ignoreIncludes, linguistics); } - public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas) { - this.config = config; - this.ignoreAutomatas = ignoreAutomatas; + public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas, Linguistics linguistics) { + this(config, ignoreAutomatas, false, linguistics); } - public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas, boolean ignoreIncludes) { + public RuleImporter(SemanticRulesConfig config, + boolean ignoreAutomatas, + boolean ignoreIncludes, + Linguistics linguistics) { this.config = config; this.ignoreAutomatas = ignoreAutomatas; this.ignoreIncludes = ignoreIncludes; + this.linguistics = linguistics; } /** @@ -91,33 +88,18 @@ public class RuleImporter { * @throws ParseException if the file does not contain a valid semantic rule set */ public RuleBase importFile(String fileName, String automataFile) throws IOException, ParseException { - return importFile(fileName, automataFile, null); - } - - /** - * Imports semantic rules from a file - * - * @param fileName the rule file to use - * @param automataFile the automata file to use, or null to not use any - * @param ruleBase an existing rule base to import these rules into, or null to create a new - * @throws java.io.IOException if the file can not be read for some reason - * @throws ParseException if the file does not contain a valid semantic rule set - */ - public RuleBase importFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { - ruleBase = privateImportFile(fileName, automataFile, ruleBase); + var ruleBase = privateImportFile(fileName, automataFile); ruleBase.initialize(); return ruleBase; } - public RuleBase privateImportFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { + public RuleBase privateImportFile(String fileName, String automataFile) throws IOException, ParseException { BufferedReader reader = null; try { reader = IOUtils.createReader(fileName, "utf-8"); File file = new File(fileName); String absoluteFileName = file.getAbsolutePath(); - if (ruleBase == null) - ruleBase = new RuleBase(); - ruleBase.setName(stripLastName(file.getName())); + var ruleBase = new RuleBase(stripLastName(file.getName()), linguistics); privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase); return ruleBase; } @@ -157,18 +139,17 @@ public class RuleImporter { /** Returns an unitialized rule base */ private RuleBase privateImportFromDirectory(String ruleBaseName, RuleBase ruleBase) throws IOException, ParseException { - RuleBase include = new RuleBase(); String includeDir = new File(ruleBase.getSource()).getParentFile().getAbsolutePath(); if (!ruleBaseName.endsWith(".sr")) ruleBaseName = ruleBaseName + ".sr"; File importFile = new File(includeDir, ruleBaseName); if ( ! importFile.exists()) throw new IOException("No file named '" + shortenPath(importFile.getPath()) + "'"); - return privateImportFile(importFile.getPath(), null, include); + return privateImportFile(importFile.getPath(), null); } /** Returns an unitialized rule base */ - private RuleBase privateImportFromConfig(String ruleBaseName) throws IOException, ParseException { + private RuleBase privateImportFromConfig(String ruleBaseName) throws ParseException { SemanticRulesConfig.Rulebase ruleBaseConfig = findRuleBaseConfig(config,ruleBaseName); if (ruleBaseConfig == null) ruleBaseConfig = findRuleBaseConfig(config, stripLastName(ruleBaseName)); @@ -224,8 +205,7 @@ public class RuleImporter { /** Imports an unitialized rule base */ public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException { if (config == null) throw new IllegalStateException("Must initialize with config if importing from config"); - RuleBase ruleBase = new RuleBase(); - ruleBase.setName(ruleBaseConfig.name()); + RuleBase ruleBase = new RuleBase(ruleBaseConfig.name(), linguistics); return privateImportFromReader(new StringReader(ruleBaseConfig.rules()), "semantic-rules.cfg", ruleBaseConfig.automata(),ruleBase); @@ -253,14 +233,10 @@ public class RuleImporter { /** Returns an unitialized rule base */ public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { try { - if (ruleBase == null) { - ruleBase = new RuleBase(); - if (sourceName == null) - sourceName = "anonymous"; - ruleBase.setName(sourceName); - } + if (ruleBase == null) + ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName, linguistics); ruleBase.setSource(sourceName.replace('\\', '/')); - new SemanticsParser(reader).semanticRules(ruleBase, this); + new SemanticsParser(reader, linguistics).semanticRules(ruleBase, this); if (automataFile != null && !automataFile.isEmpty()) ruleBase.setAutomataFile(automataFile.replace('\\', '/')); return ruleBase; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java index f9d968a3a4d..a8167fd2001 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.semantics; import com.google.inject.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.language.Linguistics; import com.yahoo.prelude.ConfigurationException; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -13,7 +14,9 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; @@ -38,7 +41,7 @@ public class SemanticSearcher extends Searcher { /** Creates a semantic searcher using the given default rule base */ public SemanticSearcher(RuleBase ruleBase) { - this(Collections.singletonList(ruleBase)); + this(List.of(ruleBase)); defaultRuleBase = ruleBase; } @@ -47,8 +50,8 @@ public class SemanticSearcher extends Searcher { } @Inject - public SemanticSearcher(SemanticRulesConfig config) { - this(toList(config)); + public SemanticSearcher(SemanticRulesConfig config, Linguistics linguistics) { + this(toList(config, linguistics)); } public SemanticSearcher(List<RuleBase> ruleBases) { @@ -59,9 +62,9 @@ public class SemanticSearcher extends Searcher { } } - private static List<RuleBase> toList(SemanticRulesConfig config) { + private static List<RuleBase> toList(SemanticRulesConfig config, Linguistics linguistics) { try { - RuleImporter ruleImporter = new RuleImporter(config); + RuleImporter ruleImporter = new RuleImporter(config, linguistics); List<RuleBase> ruleBaseList = new java.util.ArrayList<>(); for (SemanticRulesConfig.Rulebase ruleBaseConfig : config.rulebase()) { RuleBase ruleBase = ruleImporter.importConfig(ruleBaseConfig); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/benchmark/RuleBaseBenchmark.java b/container-search/src/main/java/com/yahoo/prelude/semantics/benchmark/RuleBaseBenchmark.java index 938d12b271b..75b6e831983 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/benchmark/RuleBaseBenchmark.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/benchmark/RuleBaseBenchmark.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.Iterator; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleImporter; @@ -27,7 +28,7 @@ public class RuleBaseBenchmark { fsaFile = null; } } - RuleBase ruleBase = new RuleImporter().importFile(ruleBaseFile,fsaFile); + RuleBase ruleBase = new RuleImporter(new SimpleLinguistics()).importFile(ruleBaseFile, fsaFile); ArrayList<String> queries = new ArrayList<>(); BufferedReader reader = new BufferedReader(new FileReader(queryFile)); String line; @@ -35,7 +36,7 @@ public class RuleBaseBenchmark { queries.add(line); } Date start = new Date(); - for (int i=0;i<iterations;i++){ + for (int i=0; i<iterations; i++){ for (Iterator<String> iter = queries.iterator(); iter.hasNext(); ){ String queryString = iter.next(); Query query = new Query("?query="+queryString); @@ -43,7 +44,7 @@ public class RuleBaseBenchmark { } } Date end = new Date(); - long elapsed = end.getTime()-start.getTime(); + long elapsed = end.getTime() - start.getTime(); System.out.print("BENCHMARK: rulebase=" + ruleBaseFile + "\n fsa=" + fsaFile + "\n queries=" + queryFile + diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleBaseLinguistics.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleBaseLinguistics.java new file mode 100644 index 00000000000..c5519632d6d --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleBaseLinguistics.java @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.engine; + +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.StemList; +import com.yahoo.language.process.StemMode; + +import java.util.List; +import java.util.Objects; + +/** + * Linguistics for a rule base + * + * @author bratseth + */ +public class RuleBaseLinguistics { + + private final StemMode stemMode; + private final Language language; + private final Linguistics linguistics; + + /** Creates a rule base with default settings */ + public RuleBaseLinguistics(Linguistics linguistics) { + this(StemMode.BEST, Language.ENGLISH, linguistics); + } + + + public RuleBaseLinguistics(StemMode stemMode, Language language, Linguistics linguistics) { + this.stemMode = Objects.requireNonNull(stemMode); + this.language = Objects.requireNonNull(language); + this.linguistics = Objects.requireNonNull(linguistics); + } + + public RuleBaseLinguistics withStemMode(StemMode stemMode) { + return new RuleBaseLinguistics(stemMode, language, linguistics); + } + + public RuleBaseLinguistics withLanguage(Language language) { + return new RuleBaseLinguistics(stemMode, language, linguistics); + } + + public Linguistics linguistics() { return linguistics; } + + /** Processes this term according to the linguistics of this rule base */ + public String process(String term) { + if (stemMode == StemMode.NONE) return term; + List<StemList> stems = linguistics.getStemmer().stem(term, StemMode.BEST, language); + if (stems.isEmpty()) return term; + if (stems.get(0).isEmpty()) return term; + return stems.get(0).get(0); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java index e7ed05730cb..dd6610d1184 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java @@ -17,7 +17,7 @@ import java.util.ListIterator; */ public class RuleEngine { - private RuleBase rules; + private final RuleBase rules; public RuleEngine(RuleBase rules) { this.rules=rules; @@ -38,7 +38,6 @@ public class RuleEngine { boolean matchedAnything = false; Evaluation evaluation = new Evaluation(query, traceLevel); - evaluation.setStemming(rules.getStemming()); if (traceLevel >= 2) evaluation.trace(2,"Evaluating query '" + evaluation.getQuery().getModel().getQueryTree().getRoot() + "':"); for (ListIterator<ProductionRule> i = rules.ruleIterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralCondition.java index 42bf0560726..b85dd892047 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralCondition.java @@ -4,7 +4,7 @@ package com.yahoo.prelude.semantics.rule; import com.yahoo.prelude.semantics.engine.RuleEvaluation; /** - * A condition which is always true, and which has it's own value as return value + * A condition which is always true, and which has its own value as return value * * @author bratseth */ diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java index b2592a36353..a267d274d5a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java @@ -14,9 +14,9 @@ public class NamedCondition { private Condition condition; - public NamedCondition(String name,Condition condition) { - this.conditionName=name; - this.condition=condition; + public NamedCondition(String name, Condition condition) { + this.conditionName = name; + this.condition = condition; } public String getName() { return conditionName; } @@ -28,18 +28,18 @@ public class NamedCondition { public void setCondition(Condition condition) { this.condition = condition; } public boolean matches(RuleEvaluation e) { - if (e.getTraceLevel()>=3) { + if (e.getTraceLevel() >= 3) { e.trace(3,"Evaluating '" + this + "' at " + e.currentItem()); e.indentTrace(); } boolean matches=condition.matches(e); - if (e.getTraceLevel()>=3) { + if (e.getTraceLevel() >= 3) { e.unindentTrace(); if (matches) e.trace(3,"Matched '" + this + "' at " + e.previousItem()); - else if (e.getTraceLevel()>=4) + else if (e.getTraceLevel() >= 4) e.trace(4,"Did not match '" + this + "' at " + e.currentItem()); } return matches; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamespaceProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamespaceProduction.java index 099a8562ece..e6f32a83dd9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamespaceProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamespaceProduction.java @@ -18,13 +18,13 @@ public class NamespaceProduction extends Production { private String key; /** The value to set in the namespace */ - private String value=null; + private String value; /** Creates a produced template term with no label and the default type */ - public NamespaceProduction(String namespace,String key,String value) { + public NamespaceProduction(String namespace, String key, String value) { setNamespace(namespace); - this.key=key; - this.value=value; + this.key = key; + this.value = value; } public String getNamespace() { return namespace; } @@ -44,7 +44,7 @@ public class NamespaceProduction extends Production { public void setValue(String value) { this.value = value; } - public void produce(RuleEvaluation e,int offset) { + public void produce(RuleEvaluation e, int offset) { e.getEvaluation().getQuery().properties().set(key, value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java index b36744dc397..af7abf325e7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java @@ -12,7 +12,7 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; /** - * A term produced by a production rule which takes it's actual term value + * A term produced by a production rule which takes its actual term value * from one or more terms matched in the condition * * @author bratseth diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java index 38d1fc9b83b..a2bbf72a53b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java @@ -3,6 +3,7 @@ package com.yahoo.prelude.semantics.rule; import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.semantics.engine.NameSpace; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.semantics.engine.RuleEvaluation; /** @@ -12,42 +13,35 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; */ public class TermCondition extends Condition { - private String term, termPlusS; + private final RuleBaseLinguistics linguistics; + private final String originalTerm; + private final String term; - /** Creates an invalid term */ - public TermCondition() { } - - public TermCondition(String term) { - this(null,term); + public TermCondition(String term, RuleBaseLinguistics linguistics) { + this(null, term, linguistics); } - public TermCondition(String label, String term) { + public TermCondition(String label, String term, RuleBaseLinguistics linguistics) { super(label); - this.term = term; - termPlusS = term + "s"; - } - - public String getTerm() { return term; } - - public void setTerm(String term) { - this.term = term; - termPlusS = term + "s"; + this.linguistics = linguistics; + this.originalTerm = term; + this.term = linguistics.process(term); } protected boolean doesMatch(RuleEvaluation e) { // TODO: Move this into the respective namespaces when query becomes one */ if (getNameSpace() != null) { NameSpace nameSpace = e.getEvaluation().getNameSpace(getNameSpace()); - return nameSpace.matches(term, e); + return nameSpace.matches(originalTerm, e); // No processing of terms in namespaces } else { if (e.currentItem() == null) return false; if ( ! labelMatches(e)) return false; - String matchedValue = termMatches(e.currentItem().getItem(), e.getEvaluation().getStemming()); - boolean matches = matchedValue!=null && labelMatches(e.currentItem().getItem(), e); + boolean matches = labelMatches(e.currentItem().getItem(), e) && + linguistics.process(e.currentItem().getItem().stringValue()).equals(term); if ((matches && !e.isInNegation() || (!matches && e.isInNegation()))) { - e.addMatch(e.currentItem(), matchedValue); + e.addMatch(e.currentItem(), originalTerm); e.setValue(term); e.next(); } @@ -55,34 +49,6 @@ public class TermCondition extends Condition { } } - /** Returns a non-null replacement term if there is a match, null otherwise */ - private String termMatches(TermItem queryTerm, boolean stemming) { - String queryTermString = queryTerm.stringValue(); - - // The terms are the same - boolean matches = queryTermString.equals(term); - if (matches) return term; - - if (stemming) - if (termMatchesWithStemming(queryTermString)) return term; - - return null; - } - - private boolean termMatchesWithStemming(String queryTermString) { - if (queryTermString.length() < 3) return false; // Don't stem very short terms - - // The query term minus s is the same - boolean matches = queryTermString.equals(termPlusS); - if (matches) return true; - - // The query term plus s is the same - matches = term.equals(queryTermString + "s"); - if (matches) return true; - - return false; - } - public String toInnerString() { return getLabelString() + term; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java index db8d4b42521..29e4982ac17 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java @@ -15,7 +15,7 @@ import com.yahoo.protect.Validator; public abstract class TermProduction extends Production { /** The label of this term, or null if none */ - private String label = null; + private String label; /** The type of term to produce */ private TermType termType; @@ -62,7 +62,7 @@ public abstract class TermProduction extends Production { protected void insertMatch(RuleEvaluation e, Match matched, Item newItem, int offset) { if (getWeight() != 100) newItem.setWeight(getWeight()); - int insertPosition = matched.getPosition()+offset; + int insertPosition = matched.getPosition() + offset; // This check is necessary (?) because earlier items may have been removed // after we recorded the match position. It is sort of hackish. A cleaner diff --git a/container-search/src/main/java/com/yahoo/search/query/Sorting.java b/container-search/src/main/java/com/yahoo/search/query/Sorting.java index a98ee44cb59..f1da48c1e08 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Sorting.java +++ b/container-search/src/main/java/com/yahoo/search/query/Sorting.java @@ -323,7 +323,7 @@ public class Sorting implements Cloneable { public String getLocale() { return locale; } public Strength getStrength() { return strength; } - public Collator getCollator() { return collator; } + Collator getCollator() { return collator; } public String getDecomposition() { return (collator.getDecomposition() == Collator.CANONICAL_DECOMPOSITION) ? "CANONICAL_DECOMPOSITION" : "NO_DECOMPOSITION"; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 72a8b02a960..b1d64329927 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -23,6 +23,8 @@ import java.util.Set; /** * A wrapper for structured data representing feature values: A map of floats and tensors. * This class is immutable but not thread safe. + * + * @author bratseth */ public class FeatureData implements Inspectable, JsonProducer { diff --git a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj index d79f78ef896..46117374e59 100644 --- a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj +++ b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj @@ -6,7 +6,6 @@ options { CACHE_TOKENS = true; DEBUG_PARSER = false; ERROR_REPORTING = true; - STATIC = false; UNICODE_INPUT = true; } @@ -15,12 +14,23 @@ PARSER_BEGIN(SemanticsParser) package com.yahoo.prelude.semantics.parser; import com.yahoo.javacc.UnicodeUtilities; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.Linguistics; +import com.yahoo.language.Language; import com.yahoo.prelude.semantics.*; import com.yahoo.prelude.semantics.rule.*; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.query.TermType; public class SemanticsParser { + private RuleBaseLinguistics linguistics; + + public SemanticsParser(java.io.Reader stream, Linguistics linguistics) { + this(stream); + this.linguistics = new RuleBaseLinguistics(linguistics); + } + } PARSER_END(SemanticsParser) @@ -77,6 +87,7 @@ TOKEN : <SMALLER: "<"> | <SMALLEREQUALS: "<="> | <STEMMINGDIRECTIVE: "@stemming"> | + <LANGUAGEDIRECTIVE: "@language"> | <SUPERDIRECTIVE: "@super"> | <IDENTIFIER: (~[ "\u0000"-"\u002f","\u003a"-"\u003f","\u005b"-"\u005d","\u007b"-"\u00a7","\u00a9","\u00ab"-"\u00ae","\u00b0"-"\u00b3","\u00b6"-"\u00b7","\u00b9","\u00bb"-"\u00bf", @@ -114,16 +125,20 @@ RuleBase semanticRules(RuleBase rules,RuleImporter importer) : // ---------------------------------- Directive --------------------------------------- -RuleBase directive(RuleBase rules,RuleImporter importer) : +RuleBase directive(RuleBase rules, RuleImporter importer) : { String name; } { - ( includeDirective(rules,importer) | defaultDirective(rules) | automataDirective(rules,importer) | stemmingDirective(rules) ) + ( includeDirective(rules, importer) | + defaultDirective(rules) | + automataDirective(rules, importer) | + stemmingDirective(rules) | + languageDirective(rules) ) { return rules; } } -void includeDirective(RuleBase rules,RuleImporter importer) : +void includeDirective(RuleBase rules, RuleImporter importer) : { String name; } @@ -131,25 +146,24 @@ void includeDirective(RuleBase rules,RuleImporter importer) : <INCLUDEDIRECTIVE> <LEFTBRACE> name=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? { try { - importer.include(name,rules); + importer.include(name, rules); } catch (java.io.IOException e) { - ParseException ep=new ParseException("Could not read included rule base '" + - name + "'"); + ParseException ep=new ParseException("Could not read included rule base '" + name + "'"); ep.initCause(e); throw ep; } } } -void automataDirective(RuleBase rules,RuleImporter importer) : +void automataDirective(RuleBase rules, RuleImporter importer) : { String name; } { - <AUTOMATADIRECTIVE> <LEFTBRACE> name=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? + <AUTOMATADIRECTIVE> <LEFTBRACE> name = stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? { - importer.setAutomata(rules,name); + importer.setAutomata(rules, name); } } @@ -168,9 +182,20 @@ void stemmingDirective(RuleBase rules) : String booleanString; } { - <STEMMINGDIRECTIVE> <LEFTBRACE> booleanString=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? + <STEMMINGDIRECTIVE> <LEFTBRACE> booleanString = stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? + { + linguistics = linguistics.withStemMode(Boolean.parseBoolean(booleanString) ? StemMode.BEST : StemMode.NONE); + } +} + +void languageDirective(RuleBase rules) : +{ + String languageString; +} +{ + <LANGUAGEDIRECTIVE> <LEFTBRACE> languageString = stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? { - rules.setStemming(Boolean.parseBoolean(booleanString)); + linguistics = linguistics.withLanguage(Language.from(languageString)); } } @@ -183,10 +208,10 @@ void productionRule(RuleBase rules) : ProductionList production=null; } { - condition=topLevelCondition() rule=productionRuleType() ( production=productionList() )? <SEMICOLON> + condition = topLevelCondition() rule = productionRuleType() ( production = productionList() )? <SEMICOLON> { rule.setCondition(condition); - if (production!=null) rule.setProduction(production); + if (production != null) rule.setProduction(production); rules.addRule(rule); } } @@ -201,16 +226,16 @@ ProductionRule productionRuleType() : ProductionList productionList() : { - ProductionList productionList=new ProductionList(); + ProductionList productionList = new ProductionList(); Production production; int weight=100; } { - ( production=production() (<EXCLAMATION> weight=number())? + ( production = production() (<EXCLAMATION> weight = number())? { production.setWeight(weight); productionList.addProduction(production); - weight=100; + weight = 100; } (<NL>)* ) + { return productionList; } @@ -221,7 +246,7 @@ Production production() : Production production; } { - ( LOOKAHEAD(2) production=namespaceProduction() | production=termProduction() ) + ( LOOKAHEAD(2) production = namespaceProduction() | production = termProduction() ) { return production; } } @@ -229,12 +254,12 @@ TermProduction termProduction() : { TermProduction termProduction; TermType termType; - String label=null; + String label = null; } { - termType=termType() - ( LOOKAHEAD(2) label=label() )? - ( termProduction=nonphraseTermProduction() | termProduction=phraseProduction() ) + termType = termType() + ( LOOKAHEAD(2) label = label() )? + ( termProduction = nonphraseTermProduction() | termProduction = phraseProduction() ) { termProduction.setLabel(label); @@ -248,8 +273,8 @@ TermProduction nonphraseTermProduction() : TermProduction termProduction; } { - ( termProduction=referenceTermProduction() | - termProduction=literalTermProduction() ) + ( termProduction = referenceTermProduction() | + termProduction = literalTermProduction() ) { return termProduction; } @@ -257,14 +282,14 @@ TermProduction nonphraseTermProduction() : LiteralPhraseProduction phraseProduction() : { - LiteralPhraseProduction phraseProduction=new LiteralPhraseProduction(); - String term=null; + LiteralPhraseProduction phraseProduction = new LiteralPhraseProduction(); + String term = null; } { <QUOTE> ( - term=identifier() + term = identifier() { phraseProduction.addTerm(term); } )+ <QUOTE> @@ -277,11 +302,11 @@ NamespaceProduction namespaceProduction() : { String namespace; String key; - String value=null; + String value = null; } { - namespace=identifier() <DOT> key=stringOrLiteral() <EQUALS> value=identifierOrLiteral() - { return new NamespaceProduction(namespace,key,value); } + namespace = identifier() <DOT> key = stringOrLiteral() <EQUALS> value = identifierOrLiteral() + { return new NamespaceProduction(namespace, key, value); } } ReferenceTermProduction referenceTermProduction() : @@ -289,7 +314,7 @@ ReferenceTermProduction referenceTermProduction() : String reference; } { - <LEFTSQUAREBRACKET> reference=referenceIdentifier() <RIGHTSQUAREBRACKET> + <LEFTSQUAREBRACKET> reference = referenceIdentifier() <RIGHTSQUAREBRACKET> { return new ReferenceTermProduction(reference); } } @@ -298,7 +323,7 @@ LiteralTermProduction literalTermProduction() : String literal; } { - literal=identifier() + literal = identifier() { return new LiteralTermProduction(literal); } } @@ -319,7 +344,7 @@ String referenceIdentifier() : String reference; } { - ( reference=identifier() { return reference; } ) + ( reference = identifier() { return reference; } ) | ( <ELLIPSIS> { return "..."; } ) } @@ -332,25 +357,25 @@ void namedCondition(RuleBase rules) : Condition condition; } { - <LEFTSQUAREBRACKET> conditionName=identifier() <RIGHTSQUAREBRACKET> <CONDITION> condition=topLevelCondition() <SEMICOLON> - { rules.addCondition(new NamedCondition(conditionName,condition)); } + <LEFTSQUAREBRACKET> conditionName = identifier() <RIGHTSQUAREBRACKET> <CONDITION> condition = topLevelCondition() <SEMICOLON> + { rules.addCondition(new NamedCondition(conditionName, condition)); } } Condition topLevelCondition() : { Condition condition; - boolean startAnchor=false; - boolean endAnchor=false; + boolean startAnchor = false; + boolean endAnchor = false; } { - ( <DOT> { startAnchor=true; } )? + ( <DOT> { startAnchor = true; } )? ( - LOOKAHEAD(3) condition=choiceCondition() | - LOOKAHEAD(3) condition=sequenceCondition() + LOOKAHEAD(3) condition = choiceCondition() | + LOOKAHEAD(3) condition = sequenceCondition() ) - ( LOOKAHEAD(2) <DOT> { endAnchor=true; } )? + ( LOOKAHEAD(2) <DOT> { endAnchor = true; } )? { - condition.setAnchor(Condition.Anchor.create(startAnchor,endAnchor)); + condition.setAnchor(Condition.Anchor.create(startAnchor, endAnchor)); return condition; } } @@ -361,8 +386,8 @@ Condition condition() : } { ( - ( LOOKAHEAD(3) condition=choiceCondition() - | condition=terminalCondition() ) + ( LOOKAHEAD(3) condition = choiceCondition() + | condition = terminalCondition() ) { return condition; } @@ -374,8 +399,8 @@ Condition terminalOrSequenceCondition() : Condition condition; } { - ( LOOKAHEAD(3) condition=sequenceCondition() | - condition=terminalCondition() ) + ( LOOKAHEAD(3) condition = sequenceCondition() | + condition = terminalCondition() ) { return condition; } } @@ -384,20 +409,20 @@ Condition terminalCondition() : Condition condition; } { - ( condition=notCondition() | condition=terminalOrComparisonCondition() ) + ( condition = notCondition() | condition = terminalOrComparisonCondition() ) { return condition; } } Condition terminalOrComparisonCondition() : { - Condition condition,rightCondition; + Condition condition, rightCondition; String comparison; } { - condition=reallyTerminalCondition() - ( comparison=comparison() ( LOOKAHEAD(2) rightCondition=nestedCondition() | rightCondition=reallyTerminalCondition() ) -// ( comparison=comparison() rightCondition=condition() - { condition=new ComparisonCondition(condition,comparison,rightCondition); } + condition = reallyTerminalCondition() + ( comparison = comparison() ( LOOKAHEAD(2) rightCondition = nestedCondition() | rightCondition = reallyTerminalCondition() ) +// ( comparison = comparison() rightCondition = condition() + { condition = new ComparisonCondition(condition, comparison, rightCondition); } ) ? { return condition; } @@ -405,10 +430,10 @@ Condition terminalOrComparisonCondition() : Condition reallyTerminalCondition() : { - String label=null; - String context=null; - String nameSpace=null; - Condition condition=null; + String label = null; + String context = null; + String nameSpace = null; + Condition condition = null; } { // This body looks like this to distinguish these two cases @@ -416,20 +441,20 @@ Condition reallyTerminalCondition() : // condition . (end anchor) ( LOOKAHEAD(8) ( - ( LOOKAHEAD(2) context=context() )? - ( nameSpace=nameSpace() ) - ( LOOKAHEAD(2) label=label() )? - condition=terminalConditionBody() + ( LOOKAHEAD(2) context = context() )? + ( nameSpace = nameSpace() ) + ( LOOKAHEAD(2) label = label() )? + condition = terminalConditionBody() ) | ( - ( LOOKAHEAD(2) context=context() )? - ( LOOKAHEAD(2) label=label() )? - condition=terminalConditionBody() + ( LOOKAHEAD(2) context = context() )? + ( LOOKAHEAD(2) label = label() )? + condition = terminalConditionBody() ) ) { - if (context!=null) + if (context != null) condition.setContextName(context); condition.setLabel(label); condition.setNameSpace(nameSpace); @@ -440,18 +465,18 @@ Condition reallyTerminalCondition() : Condition terminalConditionBody() : { - Condition condition=null; + Condition condition = null; } { ( - LOOKAHEAD(2) condition=conditionReference() | - condition=termCondition() | - condition=nestedCondition() | - condition=nonReferableEllipsisCondition() | - condition=referableEllipsisCondition() | - condition=superCondition() | - condition=literalCondition() | - condition=compositeItemCondition()) + LOOKAHEAD(2) condition = conditionReference() | + condition = termCondition() | + condition = nestedCondition() | + condition = nonReferableEllipsisCondition() | + condition = referableEllipsisCondition() | + condition = superCondition() | + condition = literalCondition() | + condition = compositeItemCondition()) { return condition; } } @@ -460,7 +485,7 @@ Condition notCondition() : Condition condition; } { - <EXCLAMATION> condition=terminalOrComparisonCondition() + <EXCLAMATION> condition = terminalOrComparisonCondition() { return new NotCondition(condition); } } @@ -470,7 +495,7 @@ ConditionReference conditionReference() : String conditionName; } { - <LEFTSQUAREBRACKET> conditionName=identifier() <RIGHTSQUAREBRACKET> + <LEFTSQUAREBRACKET> conditionName = identifier() <RIGHTSQUAREBRACKET> { return new ConditionReference(conditionName); } } @@ -494,23 +519,23 @@ Condition nestedCondition() : Condition condition; } { - <LEFTBRACE> condition=choiceCondition() <RIGHTBRACE> + <LEFTBRACE> condition = choiceCondition() <RIGHTBRACE> { return condition; } } Condition sequenceCondition() : { - SequenceCondition sequenceCondition=new SequenceCondition(); + SequenceCondition sequenceCondition = new SequenceCondition(); Condition condition; } { - condition=terminalCondition() + condition = terminalCondition() { sequenceCondition.addCondition(condition); } - ( LOOKAHEAD(2) condition=terminalCondition() + ( LOOKAHEAD(2) condition = terminalCondition() { sequenceCondition.addCondition(condition); } )* { - if (sequenceCondition.conditionSize()==1) + if (sequenceCondition.conditionSize() == 1) return sequenceCondition.removeCondition(0); else return sequenceCondition; @@ -519,17 +544,17 @@ Condition sequenceCondition() : Condition choiceCondition() : { - ChoiceCondition choiceCondition=new ChoiceCondition(); + ChoiceCondition choiceCondition = new ChoiceCondition(); Condition condition; } { - condition=terminalOrSequenceCondition() + condition = terminalOrSequenceCondition() { choiceCondition.addCondition(condition); } - ( LOOKAHEAD(3) (<NL>)* <COMMA> (<NL>)* condition=terminalOrSequenceCondition() + ( LOOKAHEAD(3) (<NL>)* <COMMA> (<NL>)* condition = terminalOrSequenceCondition() { choiceCondition.addCondition(condition); } ) * { - if (choiceCondition.conditionSize()==1) + if (choiceCondition.conditionSize() == 1) return choiceCondition.removeCondition(0); else return choiceCondition; @@ -542,7 +567,7 @@ TermCondition termCondition() : } { ( str = identifier() ) - { return new TermCondition(str); } + { return new TermCondition(str, linguistics); } } SuperCondition superCondition() : { } @@ -566,7 +591,7 @@ CompositeItemCondition compositeItemCondition() : CompositeItemCondition compositeItemCondition = new CompositeItemCondition(); } { - ( <QUOTE> ( condition=terminalConditionBody() { compositeItemCondition.addCondition(condition); } ) <QUOTE> ) + ( <QUOTE> ( condition = terminalConditionBody() { compositeItemCondition.addCondition(condition); } ) <QUOTE> ) { return compositeItemCondition; } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java index ca5bb4d4cd2..bf99a709df3 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.semantics.parser.test; import java.util.Iterator; import com.yahoo.javacc.UnicodeUtilities; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleImporter; import com.yahoo.prelude.semantics.parser.ParseException; @@ -24,8 +25,8 @@ public class SemanticsParserTestCase { @Test public void testRuleReading() throws java.io.IOException, ParseException { - RuleBase rules=new RuleImporter().importFile(ROOT + "rules.sr"); - Iterator<?> i=rules.ruleIterator(); + RuleBase rules = new RuleImporter(new SimpleLinguistics()).importFile(ROOT + "rules.sr"); + Iterator<?> i = rules.ruleIterator(); assertEquals("[listing] [preposition] [place] -> listing:[listing] place:[place]!150", i.next().toString()); assertEquals("[listing] [place] +> place:[place]", @@ -36,10 +37,10 @@ public class SemanticsParserTestCase { i.next().toString()); assertEquals("digital camera -> digicamera", i.next().toString()); - assertEquals("(parameter.ranking='cat'), (parameter.ranking='cat0') -> one",i.next().toString()); + assertEquals("(parameter.ranking='cat'), (parameter.ranking='cat0') -> one", i.next().toString()); assertFalse(i.hasNext()); - i=rules.conditionIterator(); + i = rules.conditionIterator(); assertEquals("[listing] :- restaurant, shop, cafe, hotel", i.next().toString()); assertEquals("[preposition] :- in, at, near", @@ -53,7 +54,7 @@ public class SemanticsParserTestCase { assertFalse(i.hasNext()); assertTrue(rules.isDefault()); - assertEquals(ROOT + "semantics.fsa",rules.getAutomataFile()); + assertEquals(ROOT + "semantics.fsa", rules.getAutomataFile()); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java index ac1791ae91a..394752f8aa1 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java @@ -30,7 +30,7 @@ public class BacktrackingTestCase { static { try { - searcher = new SemanticSearcher(new RuleImporter().importFile(root + "backtrackingrules.sr")); + searcher = new SemanticSearcher(new RuleImporter(new SimpleLinguistics()).importFile(root + "backtrackingrules.sr")); } catch (Exception e) { throw new RuntimeException(e); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java index 86ee9b5948b..eb69372c22b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.test; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.engine.Evaluation; @@ -24,15 +26,17 @@ public class ConditionTestCase { @Test public void testTermCondition() { - TermCondition term=new TermCondition("foo"); - Query query=new Query("?query=foo"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term = new TermCondition("foo", linguistics); + Query query = new Query("?query=foo"); assertTrue(term.matches(new Evaluation(query).freshRuleEvaluation())); } @Test public void testSequenceCondition() { - TermCondition term1 = new TermCondition("foo"); - TermCondition term2 = new TermCondition("bar"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term1 = new TermCondition("foo", linguistics); + TermCondition term2 = new TermCondition("bar",linguistics); SequenceCondition sequence = new SequenceCondition(); sequence.addCondition(term1); sequence.addCondition(term2); @@ -46,8 +50,9 @@ public class ConditionTestCase { @Test public void testChoiceCondition() { - TermCondition term1 = new TermCondition("foo"); - TermCondition term2 = new TermCondition("bar"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term1 = new TermCondition("foo", linguistics); + TermCondition term2 = new TermCondition("bar", linguistics); ChoiceCondition choice = new ChoiceCondition(); choice.addCondition(term1); choice.addCondition(term2); @@ -61,7 +66,8 @@ public class ConditionTestCase { @Test public void testNamedConditionReference() { - TermCondition term = new TermCondition("foo"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term = new TermCondition("foo", linguistics); NamedCondition named = new NamedCondition("cond",term); ConditionReference reference = new ConditionReference("cond"); @@ -69,8 +75,7 @@ public class ConditionTestCase { ProductionRule rule = new ReplacingProductionRule(); rule.setCondition(reference); rule.setProduction(new ProductionList()); - RuleBase ruleBase = new RuleBase(); - ruleBase.setName("test"); + RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java index 80c9e898302..6d5b9459833 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java @@ -37,7 +37,7 @@ public class ConfigurationTestCase { static { semanticRulesConfig = new ConfigGetter<>(SemanticRulesConfig.class).getConfig("file:" + root + "semantic-rules.cfg"); - searcher=new SemanticSearcher(semanticRulesConfig); + searcher = new SemanticSearcher(semanticRulesConfig, new SimpleLinguistics()); } protected void assertSemantics(String result, String input, String baseName) { @@ -54,46 +54,46 @@ public class ConfigurationTestCase { @Test public void testReadingConfigurationRuleBase() { - RuleBase parent=searcher.getRuleBase("parent"); + RuleBase parent = searcher.getRuleBase("parent"); assertNotNull(parent); - assertEquals("parent",parent.getName()); - assertEquals("semantic-rules.cfg",parent.getSource()); + assertEquals("parent", parent.getName()); + assertEquals("semantic-rules.cfg", parent.getSource()); } @Test - public void testParent() throws Exception { - assertSemantics("vehiclebrand:audi","audi cars","parent"); - assertSemantics("vehiclebrand:alfa","alfa bus","parent"); - assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","parent.sr"); - assertSemantics("AND vw car", "vw cars","parent"); - assertSemantics("AND skoda car", "skoda cars","parent.sr"); + public void testParent() { + assertSemantics("vehiclebrand:audi", "audi cars", "parent"); + assertSemantics("vehiclebrand:alfa", "alfa bus", "parent"); + assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "parent.sr"); + assertSemantics("AND vw car", "vw cars", "parent"); + assertSemantics("AND skoda car", "skoda cars", "parent.sr"); } @Test - public void testChild1() throws Exception { - assertSemantics("vehiclebrand:skoda","audi cars","child1.sr"); - assertSemantics("vehiclebrand:alfa", "alfa bus","child1"); - assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","child1"); - assertSemantics("vehiclebrand:skoda","vw cars","child1"); - assertSemantics("AND skoda car", "skoda cars","child1"); + public void testChild1() { + assertSemantics("vehiclebrand:skoda", "audi cars", "child1.sr"); + assertSemantics("vehiclebrand:alfa", "alfa bus", "child1"); + assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "child1"); + assertSemantics("vehiclebrand:skoda", "vw cars", "child1"); + assertSemantics("AND skoda car", "skoda cars", "child1"); } @Test - public void testChild2() throws Exception { - assertSemantics("vehiclebrand:audi","audi cars","child2"); - assertSemantics("vehiclebrand:alfa","alfa bus","child2.sr"); - assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","child2.sr"); - assertSemantics("AND vw car","vw cars","child2"); - assertSemantics("vehiclebrand:skoda","skoda cars","child2"); + public void testChild2() { + assertSemantics("vehiclebrand:audi", "audi cars", "child2"); + assertSemantics("vehiclebrand:alfa", "alfa bus", "child2.sr"); + assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "child2.sr"); + assertSemantics("AND vw car", "vw cars", "child2"); + assertSemantics("vehiclebrand:skoda", "skoda cars", "child2"); } @Test - public void testGrandchild() throws Exception { - assertSemantics("vehiclebrand:skoda","audi cars","grandchild.sr"); - assertSemantics("vehiclebrand:alfa","alfa bus","grandchild"); - assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","grandchild"); - assertSemantics("vehiclebrand:skoda","vw cars","grandchild"); - assertSemantics("vehiclebrand:skoda","skoda cars","grandchild"); + public void testGrandchild() { + assertSemantics("vehiclebrand:skoda", "audi cars", "grandchild.sr"); + assertSemantics("vehiclebrand:alfa", "alfa bus", "grandchild"); + assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "grandchild"); + assertSemantics("vehiclebrand:skoda", "vw cars", "grandchild"); + assertSemantics("vehiclebrand:skoda", "skoda cars", "grandchild"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java index fb86beaa9bc..76c8c3966b7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.test; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.semantics.RuleBaseException; import com.yahoo.prelude.semantics.RuleImporter; import com.yahoo.prelude.semantics.parser.ParseException; @@ -14,18 +15,18 @@ import static org.junit.Assert.fail; */ public class DuplicateRuleTestCase { - private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + private final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; @Test public void testDuplicateRuleBaseLoading() throws java.io.IOException, ParseException { if (System.currentTimeMillis() > 0) return; // TODO: Include this test... try { - new RuleImporter().importFile(root + "rules.sr"); + new RuleImporter(new SimpleLinguistics()).importFile(root + "rules.sr"); fail("Did not detect duplicate condition names"); } catch (RuleBaseException e) { - assertEquals("Duplicate condition 'something' in 'duplicaterules.sr'",e.getMessage()); + assertEquals("Duplicate condition 'something' in 'duplicaterules.sr'", e.getMessage()); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java index d93fd218259..e9364074281 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.StringTokenizer; import com.yahoo.component.chain.Chain; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; @@ -24,7 +25,6 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -@SuppressWarnings("deprecation") public class InheritanceTestCase { private static final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; @@ -34,10 +34,10 @@ public class InheritanceTestCase { static { try { - parent = RuleBase.createFromFile(root + "inheritingrules/parent.sr", null); - child1 = RuleBase.createFromFile(root + "inheritingrules/child1.sr", null); - child2 = RuleBase.createFromFile(root + "inheritingrules/child2.sr", null); - grandchild = RuleBase.createFromFile(root + "inheritingrules/grandchild.sr", null); + parent = RuleBase.createFromFile(root + "inheritingrules/parent.sr", null, new SimpleLinguistics()); + child1 = RuleBase.createFromFile(root + "inheritingrules/child1.sr", null, new SimpleLinguistics()); + child2 = RuleBase.createFromFile(root + "inheritingrules/child2.sr", null, new SimpleLinguistics()); + grandchild = RuleBase.createFromFile(root + "inheritingrules/grandchild.sr", null, new SimpleLinguistics()); grandchild.setDefault(true); searcher = new SemanticSearcher(parent, child1, child2, grandchild); @@ -77,7 +77,7 @@ public class InheritanceTestCase { public void testInclusionOrderAndContentDump() { StringTokenizer lines = new StringTokenizer(grandchild.toContentString(),"\n",false); assertEquals("vw -> audi", lines.nextToken()); - assertEquals("cars -> car", lines.nextToken()); + assertEquals("car -> car", lines.nextToken()); assertEquals("[brand] [vehicle] -> vehiclebrand:[brand]", lines.nextToken()); assertEquals("vehiclebrand:bmw +> expensivetv", lines.nextToken()); assertEquals("vehiclebrand:audi -> vehiclebrand:skoda", lines.nextToken()); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MusicTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MusicTestCase.java new file mode 100644 index 00000000000..006dcb3c714 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MusicTestCase.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import org.junit.Test; + +/** + * Tests the rewriting in the semanticsearcher system test + * + * @author bratseth + */ +public class MusicTestCase { + + @Test + public void testMusic() { + var tester = new RuleBaseTester("music.sr"); + tester.assertSemantics("AND song:together artist:youngbloods", "together by youngbloods"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java index cd5743c6d77..376da065f4d 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java @@ -20,66 +20,65 @@ public class ParameterTestCase extends RuleBaseAbstractTestCase { /** Tests parameter literal matching */ @Test public void testLiteralEquals() { - assertSemantics("a","a"); - assertSemantics("RANK a foo:a","a&ranking=category"); - assertSemantics("a","a&ranking=somethingelse"); - assertSemantics("a","a&otherparam=category"); + assertSemantics("a", "a"); + assertSemantics("RANK a foo:a", "a&ranking=category"); + assertSemantics("a", "a&ranking=somethingelse"); + assertSemantics("a", "a&otherparam=category"); } /** Tests parameter matching of larger */ @Test public void testLarger() { - assertSemantics("a","a"); - assertSemantics("AND a largepage","a&hits=11"); - assertSemantics("AND a largepage","a&hits=12"); + assertSemantics("a", "a"); + assertSemantics("AND a largepage", "a&hits=11"); + assertSemantics("AND a largepage", "a&hits=12"); } /** Tests parameter containment matching */ @Test public void testContainsAsList() { assertSemantics("a","a"); - assertSemantics("AND a intent:music","a&search=music"); - assertSemantics("AND a intent:music","a&search=music,books"); - assertSemantics("AND a intent:music","a&search=kanoos,music,books"); + assertSemantics("AND a intent:music", "a&search=music"); + assertSemantics("AND a intent:music", "a&search=music,books"); + assertSemantics("AND a intent:music", "a&search=kanoos,music,books"); } /** Tests parameter production */ @Test public void testParameterProduction() { - assertParameterSemantics("AND a b c","a b c","search","[letters, alphabet]"); - assertParameterSemantics("AND a c d","a c d","search","[letters, someletters]"); - assertParameterSemantics("+(AND a d e) -letter:c","a d e","search","[someletters]"); - assertParameterSemantics("AND a d f","a d f","rank-profile","foo"); - assertParameterSemantics("AND a f g","a f g","grouping.nolearning","true"); + assertParameterSemantics("AND a b c", "a b c", "search", "[letters, alphabet]"); + assertParameterSemantics("AND a c d", "a c d", "search", "[letters, someletters]"); + assertParameterSemantics("+(AND a d e) -letter:c", "a d e", "search", "[someletters]"); + assertParameterSemantics("AND a d f", "a d f", "rank-profile", "foo"); + assertParameterSemantics("AND a f g", "a f g", "grouping.nolearning", "true"); } @Test public void testMultipleAlternativeParameterValuesInCondition() { - assertInputRankParameterSemantics("one","foo","cat"); - assertInputRankParameterSemantics("one","foo","cat0"); - assertInputRankParameterSemantics("one","bar","cat"); - assertInputRankParameterSemantics("one","bar","cat0"); - assertInputRankParameterSemantics("AND one one","foo+bar","cat0"); - assertInputRankParameterSemantics("AND fuki sushi","fuki+sushi","cat0"); + assertInputRankParameterSemantics("one", "foo", "cat"); + assertInputRankParameterSemantics("one", "foo", "cat0"); + assertInputRankParameterSemantics("one", "bar", "cat"); + assertInputRankParameterSemantics("one", "bar", "cat0"); + assertInputRankParameterSemantics("AND one one", "foo+bar", "cat0"); + assertInputRankParameterSemantics("AND fuki sushi", "fuki+sushi", "cat0"); } - private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, - String rankParameterValue) { - assertInputRankParameterSemantics(producedQuery,inputQuery,rankParameterValue,0); + private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, String rankParameterValue) { + assertInputRankParameterSemantics(producedQuery, inputQuery, rankParameterValue, 0); } - private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, - String rankParameterValue,int tracelevel) { - Query query=new Query("?query=" + inputQuery + "&tracelevel=0&tracelevel.rules=" + tracelevel); + private void assertInputRankParameterSemantics(String producedQuery, String inputQuery, + String rankParameterValue, int tracelevel) { + Query query = new Query("?query=" + inputQuery + "&tracelevel=0&tracelevel.rules=" + tracelevel); query.getRanking().setProfile(rankParameterValue); query.properties().set("tracelevel.rules", tracelevel); assertSemantics(producedQuery, query); } - private void assertParameterSemantics(String producedQuery,String inputQuery, - String producedParameterName,String producedParameterValue) { - Query query=assertSemantics(producedQuery,inputQuery); - assertEquals(producedParameterValue,query.properties().getString(producedParameterName)); + private void assertParameterSemantics(String producedQuery, String inputQuery, + String producedParameterName, String producedParameterValue) { + Query query = assertSemantics(producedQuery, inputQuery); + assertEquals(producedParameterValue, query.properties().getString(producedParameterName)); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java index 8b883759215..b91e9441a2b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.test; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.engine.Evaluation; @@ -25,7 +27,8 @@ public class ProductionRuleTestCase { @Test public void testProductionRule() { - TermCondition term = new TermCondition("sony"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term = new TermCondition("sony", linguistics); NamedCondition named = new NamedCondition("brand", term); ConditionReference reference = new ConditionReference("brand"); @@ -38,8 +41,7 @@ public class ProductionRuleTestCase { rule.setProduction(productionList); // To initialize the condition reference... - RuleBase ruleBase = new RuleBase(); - ruleBase.setName("test"); + RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java index baccb73cd93..84e47edae29 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.semantics.test; import com.yahoo.component.chain.Chain; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; @@ -16,7 +17,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; /** - * Tests semantic searching + * DO NOT USE. Use RuleBaseTester instead * * @author bratseth */ @@ -37,7 +38,7 @@ public abstract class RuleBaseAbstractTestCase { try { if (automataFileName != null) automataFileName = root + automataFileName; - RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName); + RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName, new SimpleLinguistics()); return new SemanticSearcher(ruleBase); } catch (Exception e) { throw new RuleBaseException("Initialization of rule base '" + ruleBaseName + "' failed",e); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java new file mode 100644 index 00000000000..cc9e758a0e0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.opennlp.OpenNlpLinguistics; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.prelude.semantics.SemanticSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Helper for testing with a rule base. + * Replace subclassing of RuleBaseAbstractTestCase by this. + * + * @author bratseth + */ +public class RuleBaseTester { + + private final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + private final SemanticSearcher searcher; + + public RuleBaseTester(String ruleBaseName) { + this(ruleBaseName, null); + } + + public RuleBaseTester(String ruleBaseName, String automataFileName) { + searcher = createSearcher(ruleBaseName, automataFileName); + } + + private SemanticSearcher createSearcher(String ruleBaseName,String automataFileName) { + try { + if (automataFileName != null) + automataFileName = root + automataFileName; + RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName, new OpenNlpLinguistics()); + return new SemanticSearcher(ruleBase); + } catch (Exception e) { + throw new RuleBaseException("Initialization of rule base '" + ruleBaseName + "' failed", e); + } + } + + public Query assertSemantics(String result, String input) { + return assertSemantics(result, input, 0); + } + + public Query assertSemantics(String result, String input, int tracelevel) { + return assertSemantics(result, input, tracelevel, Query.Type.ALL); + } + + public Query assertSemantics(String result, String input, int tracelevel, Query.Type queryType) { + Query query = new Query("?query=" + QueryTestCase.httpEncode(input) + "&tracelevel=0&tracelevel.rules=" + tracelevel + + "&language=und&type=" + queryType); + return assertSemantics(result, query); + } + + public Query assertSemantics(String result, Query query) { + createExecution(searcher).search(query); + assertEquals(result, query.getModel().getQueryTree().getRoot().toString()); + return query; + } + + private Execution createExecution(Searcher searcher) { + return new Execution(chainedAsSearchChain(searcher), Execution.Context.createContextStub()); + } + + private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { + List<Searcher> searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java index 29cc5c6e23a..76b2d3991c1 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java @@ -142,11 +142,6 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { } @Test - public void testPluralReplaceBecomesSingular() { - assertSemantics("AND from:paris to:texas","pariss to texass"); - } - - @Test public void testOrProduction() { assertSemantics("OR something somethingelse", "something"); } @@ -155,7 +150,7 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { @Test public void testWeightedSetItem() { Query q = new Query(); - WeightedSetItem weightedSet=new WeightedSetItem("fieldName"); + WeightedSetItem weightedSet = new WeightedSetItem("fieldName"); weightedSet.addToken("a", 1); weightedSet.addToken("b", 2); q.getModel().getQueryTree().setRoot(weightedSet); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java index 6702a1ca1d9..b8efbf7422b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java @@ -4,34 +4,52 @@ package com.yahoo.prelude.semantics.test; import org.junit.Test; /** - * Tests a case reported by tularam + * Tests stemming. * * @author bratseth */ -public class StemmingTestCase extends RuleBaseAbstractTestCase { +public class StemmingTestCase { - public StemmingTestCase() { - super("stemming.sr"); + @Test + public void testRewritingDueToStemmingInQuery() { + var tester = new RuleBaseTester("stemming.sr"); + tester.assertSemantics("+(AND i:vehicle TRUE) -i:s", "i:cars -i:s"); } @Test - public void testRewritingDueToStemmingInQuery() { - assertSemantics("+(AND i:vehicle TRUE) -i:s","i:cars -i:s"); + public void testNoRewritingDueToStemmingInQueryWhenStemmingDisabled() { + var tester = new RuleBaseTester("stemming-none.sr"); + tester.assertSemantics("+i:cars -i:s", "i:cars -i:s"); } @Test public void testRewritingDueToStemmingInRule() { - assertSemantics("+(AND i:animal TRUE) -i:s","i:horse -i:s"); + var tester = new RuleBaseTester("stemming.sr"); + tester.assertSemantics("+(AND i:animal TRUE) -i:s", "i:horse -i:s"); + } + + @Test + public void testNoRewritingDueToStemmingInRuleWhenStemmingDisabled() { + var tester = new RuleBaseTester("stemming-none.sr"); + tester.assertSemantics("+i:horse -i:s", "i:horse -i:s"); } @Test public void testRewritingDueToExactMatch() { - assertSemantics("+(AND i:arts i:sciences TRUE) -i:s","i:as -i:s"); + var tester = new RuleBaseTester("stemming.sr"); + tester.assertSemantics("+(AND i:arts i:sciences TRUE) -i:s", "i:as -i:s"); + } + + @Test + public void testEnglishStemming() { + var tester = new RuleBaseTester("stemming.sr"); + tester.assertSemantics("i:drive", "i:going"); } @Test - public void testNoRewritingBecauseShortWordsAreNotStemmed() { - assertSemantics("+i:a -i:s","i:a -i:s"); + public void testFrenchStemming() { + var tester = new RuleBaseTester("stemming-french.sr"); + tester.assertSemantics("i:going", "i:going"); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/music.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/music.sr new file mode 100644 index 00000000000..5166a8b8e5d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/music.sr @@ -0,0 +1,19 @@ +## Some test rules + +# Spelling correction +bahc -> bach; + +# Stopwords +somelongstopword -> ; +[stopword] -> ; +[stopword] :- someotherlongstopword, yetanotherstopword; + +# +[song] by [artist] -> song:[song] artist:[artist]; + +[song] :- together, imagine, tinseltown; +[artist] :- youngbloods, beatles, zappa; + +# Negative +various +> -kingz; + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr new file mode 100644 index 00000000000..1ccafd04344 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@stemming(true) +@language(fr) + +i:as -> i:arts i:sciences; +i:car -> i:vehicle; +i:horses -> i:animal; +i:go -> i:drive; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr new file mode 100644 index 00000000000..44f6e40a308 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr @@ -0,0 +1,6 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@stemming(false) + +i:car -> i:vehicle; +i:horses -> i:animal; +i:go -> i:drive; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr index f68706646c2..ea73e385b3a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr @@ -1,5 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @stemming(true) + i:as -> i:arts i:sciences; i:car -> i:vehicle; i:horses -> i:animal; +i:go -> i:drive; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index a6e53fead37..5c0669ad543 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -61,6 +61,12 @@ public class ApplicationList extends AbstractFilteringList<Application, Applicat .anyMatch(deployment -> deployment.version().isBefore(version))); } + /** Returns the subset of applications with at least one declared job in deployment spec. */ + public ApplicationList withJobs() { + return matching(application -> application.deploymentSpec().steps().stream() + .anyMatch(step -> ! step.zones().isEmpty())); + } + /** Returns the subset of applications which have a project ID */ public ApplicationList withProjectId() { return matching(application -> application.projectId().isPresent()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 24117b9f55f..b5008a44c6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -94,9 +94,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { Cursor platformArray = root.setArray("versions"); var versionStatus = controller.readVersionStatus(); var systemVersion = controller.systemVersion(versionStatus); - ApplicationList applications = ApplicationList.from(controller.applications().asList()) - .matching(application -> application.deploymentSpec().steps().stream() - .anyMatch(step -> ! step.zones().isEmpty())); + ApplicationList applications = ApplicationList.from(controller.applications().asList()).withJobs(); var deploymentStatuses = controller.jobController().deploymentStatuses(applications, systemVersion); var deploymentStatistics = DeploymentStatistics.compute(versionStatus.versions().stream().map(VespaVersion::versionNumber).collect(toList()), deploymentStatuses) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 24a739d4fc4..238ab0b09fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -121,7 +121,8 @@ public class VersionStatus { List<DeploymentStatistics> deploymentStatistics = DeploymentStatistics.compute(infrastructureVersions.keySet(), controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()) - .withProjectId())); + .withProjectId() + .withJobs())); List<VespaVersion> versions = new ArrayList<>(); List<Version> releasedVersions = controller.mavenRepository().metadata().versions(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 61aab87532f..ca0db13b3d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -33,7 +33,7 @@ public class DeploymentApiTest extends ControllerContainerTest { public void testDeploymentApi() { ContainerTester tester = new ContainerTester(container, responseFiles); DeploymentTester deploymentTester = new DeploymentTester(new ControllerTester(tester)); - Version version = Version.fromString("5.0"); + Version version = Version.fromString("4.9"); deploymentTester.controllerTester().upgradeSystem(version); ApplicationPackage multiInstancePackage = new ApplicationPackageBuilder() .instances("i1,i2") @@ -42,14 +42,24 @@ public class DeploymentApiTest extends ControllerContainerTest { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") .build(); - ApplicationPackage emptyPackage = new ApplicationPackageBuilder().instances("custom").build(); + ApplicationPackage emptyPackage = new ApplicationPackageBuilder().instances("default") + .allow(ValidationId.deploymentRemoval) + .build(); - // 3 applications deploy on current system version, 1 is empty + // Deploy application without any declared jobs on the oldest version. + var oldAppWithoutDeployment = deploymentTester.newDeploymentContext("tenant4", "application4", "default"); + oldAppWithoutDeployment.submit().failDeployment(JobType.systemTest); + oldAppWithoutDeployment.submit(emptyPackage); + + // System upgrades to 5.0 for the other applications. + version = Version.fromString("5.0"); + deploymentTester.controllerTester().upgradeSystem(version); + + // 3 applications deploy on current system version. var failingApp = deploymentTester.newDeploymentContext("tenant1", "application1", "default"); var productionApp = deploymentTester.newDeploymentContext("tenant2", "application2", "i1"); var otherProductionApp = deploymentTester.newDeploymentContext("tenant2", "application2", "i2"); var appWithoutDeployments = deploymentTester.newDeploymentContext("tenant3", "application3", "default"); - var otherAppWithoutDeployment = deploymentTester.newDeploymentContext("tenant4", "application4", "custom"); failingApp.submit(applicationPackage).deploy(); productionApp.submit(multiInstancePackage).runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1); otherProductionApp.runJob(JobType.productionUsWest1); @@ -58,9 +68,6 @@ public class DeploymentApiTest extends ControllerContainerTest { appWithoutDeployments.submit(applicationPackage).deploy(); appWithoutDeployments.submit(new ApplicationPackageBuilder().allow(ValidationId.deploymentRemoval).build()); - // Deploy application without any declared jobs. - otherAppWithoutDeployment.submit(emptyPackage); - // New version released version = Version.fromString("5.1"); deploymentTester.controllerTester().upgradeSystem(version); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 32b65ac8efb..2a36c62239a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -60,7 +60,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( - "feed-sequencer-type", "LATENCY", + "feed-sequencer-type", "THROUGHPUT", List.of("baldersheim"), "2020-12-02", "2022-02-01", "Selects type of sequenced executor used for feeding in proton, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment (requires restart)", @@ -74,7 +74,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag FEED_MASTER_TASK_LIMIT = defineIntFlag( - "feed-master-task-limit", 0, + "feed-master-task-limit", 1000, List.of("geirst, baldersheim"), "2021-11-18", "2022-02-01", "The task limit used by the master thread in each document db in proton. Ignored when set to 0.", "Takes effect at redeployment", @@ -173,12 +173,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag NUM_DEPLOY_HELPER_THREADS = defineIntFlag( - "num-model-builder-threads", -1, - List.of("balder"), "2021-09-09", "2022-02-01", - "Number of threads used for speeding up building of models.", - "Takes effect on first (re)start of config server"); - public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", true, List.of("geirst"), "2021-01-27", "2022-01-31", @@ -222,34 +216,27 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( - "max-concurrent-merges-per-node", 128, + "max-concurrent-merges-per-node", 16, List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( - "max-merge-queue-size", 1024, + "max-merge-queue-size", 100, List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", "Specifies max size of merge queue.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag IGNORE_MERGE_QUEUE_LIMIT = defineFeatureFlag( - "ignore-merge-queue-limit", false, + "ignore-merge-queue-limit", true, List.of("vekterli", "geirst"), "2021-10-06", "2022-03-01", "Specifies if merges that are forwarded (chained) from another content node are always " + "allowed to be enqueued even if the queue is otherwise full.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag LARGE_RANK_EXPRESSION_LIMIT = defineIntFlag( - "large-rank-expression-limit", 8192, - List.of("baldersheim"), "2021-06-09", "2022-02-01", - "Limit for size of rank expressions distributed by filedistribution", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, List.of("geirst", "vekterli"), "2021-07-16", "2022-03-01", @@ -319,7 +306,7 @@ public class Flags { ); public static final UnboundIntFlag DISTRIBUTOR_MERGE_BUSY_WAIT = defineIntFlag( - "distributor-merge-busy-wait", 10, + "distributor-merge-busy-wait", 1, List.of("geirst", "vekterli"), "2021-10-04", "2022-03-01", "Number of seconds that scheduling of new merge operations in the distributor should be inhibited " + "towards a content node that has indicated merge busy", @@ -327,21 +314,21 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING = defineFeatureFlag( - "distributor-enhanced-maintenance-scheduling", false, + "distributor-enhanced-maintenance-scheduling", true, List.of("vekterli", "geirst"), "2021-10-14", "2022-01-31", "Enable enhanced maintenance operation scheduling semantics on the distributor", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag ASYNC_APPLY_BUCKET_DIFF = defineFeatureFlag( - "async-apply-bucket-diff", false, + "async-apply-bucket-diff", true, List.of("geirst", "vekterli"), "2021-10-22", "2022-01-31", "Whether portions of apply bucket diff handling will be performed asynchronously", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag UNORDERED_MERGE_CHAINING = defineFeatureFlag( - "unordered-merge-chaining", false, + "unordered-merge-chaining", true, List.of("vekterli", "geirst"), "2021-11-15", "2022-03-01", "Enables the use of unordered merge chains for data merge operations", "Takes effect at redeploy", diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index 31612bea983..910056286ec 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -13,6 +13,7 @@ "public java.lang.String languageCode()", "public boolean isCjk()", "public static com.yahoo.language.Language fromLanguageTag(java.lang.String)", + "public static com.yahoo.language.Language from(java.lang.String)", "public static com.yahoo.language.Language fromLocale(java.util.Locale)", "public static com.yahoo.language.Language fromEncoding(java.lang.String)" ], diff --git a/linguistics/src/main/java/com/yahoo/language/Language.java b/linguistics/src/main/java/com/yahoo/language/Language.java index 9f60985c119..e4ac280af9e 100644 --- a/linguistics/src/main/java/com/yahoo/language/Language.java +++ b/linguistics/src/main/java/com/yahoo/language/Language.java @@ -6,6 +6,7 @@ import com.yahoo.text.Lowercase; import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Objects; /** * @author Rich Pito @@ -529,10 +530,11 @@ public enum Language { } /** - * <p>Convenience method for calling <code>fromLocale(LocaleFactory.fromLanguageTag(languageTag))</code>.</p> + * Convenience method for calling <code>fromLocale(LocaleFactory.fromLanguageTag(languageTag))</code>. + * Returns UNKNOWN when passed null or an unknown language tag. * - * @param languageTag The language tag for which the <code>Language</code> to return. - * @return the corresponding <code>Language</code>, or {@link #UNKNOWN} if not known. + * @param languageTag the language tag for which the <code>Language</code> to return + * @return the corresponding <code>Language</code>, or {@link #UNKNOWN} if not known */ public static Language fromLanguageTag(String languageTag) { if (languageTag == null) return UNKNOWN; @@ -540,6 +542,21 @@ public enum Language { } /** + * Returns the Language from a language tag + * + * @param languageTag the language tag for which the <code>Language</code> to return, cannot be null + * @return the Language instance + * @throws IllegalArgumentException if the language tag is unknown + */ + public static Language from(String languageTag) { + Objects.requireNonNull(languageTag, "languageTag cannot be null"); + Language language = fromLocale(LocaleFactory.fromLanguageTag(languageTag)); + if ( ! languageTag.equalsIgnoreCase("unknown") && language == Language.UNKNOWN) + throw new IllegalArgumentException("Unknown language tag '" + languageTag + "'"); + return language; + } + + /** * <p>Returns the <code>Language</code> whose {@link #languageCode()} is equal to <code>locale.getLanguage()</code>, with * the following additions:</p> * <ul> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java new file mode 100644 index 00000000000..65d7ebaa02d --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; + +/** + * @author hakonhall + */ +public class BadTemplateException extends TemplateException { + public BadTemplateException(Cursor location, String message) { + super(message + " at " + location.calculateLocation().lineAndColumnText()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java new file mode 100644 index 00000000000..6b38835e24f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java @@ -0,0 +1,86 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * A form is an instance of a template to be filled, e.g. values set for variable sections, etc. + * + * @see Template + * @author hakonhall + */ +public class Form { + private Form parent = null; + private final CursorRange range; + private final List<Section> sections; + + private final Map<String, String> values = new HashMap<>(); + private final Map<String, ListSection> lists; + + Form(CursorRange range, List<Section> sections, Map<String, ListSection> lists) { + this.range = new CursorRange(range); + this.sections = List.copyOf(sections); + this.lists = Map.copyOf(lists); + } + + void setParent(Form parent) { this.parent = parent; } + + /** Set the value of a variable, e.g. %{=color}. */ + public Form set(String name, String value) { + values.put(name, value); + return this; + } + + /** Set the value of a variable and/or if-condition. */ + public Form set(String name, boolean value) { return set(name, Boolean.toString(value)); } + + public Form set(String name, int value) { return set(name, Integer.toString(value)); } + public Form set(String name, long value) { return set(name, Long.toString(value)); } + + public Form set(String name, String format, String first, String... rest) { + var args = new Object[1 + rest.length]; + args[0] = first; + System.arraycopy(rest, 0, args, 1, rest.length); + var value = String.format(format, args); + + return set(name, value); + } + + /** Add an instance of a list section after any previously added (for the given name) */ + public Form add(String name) { + var section = lists.get(name); + if (section == null) { + throw new NoSuchNameTemplateException(range, name); + } + return section.add(); + } + + public String render() { + var buffer = new StringBuilder((int) (range.length() * 1.2 + 128)); + appendTo(buffer); + return buffer.toString(); + } + + public void appendTo(StringBuilder buffer) { + sections.forEach(section -> section.appendTo(buffer)); + } + + /** Returns a deep copy of this. No changes to this affects the returned form, and vice versa. */ + Form copy() { + var builder = new FormBuilder(range.start()); + sections.forEach(section -> section.appendCopyTo(builder.topLevelSectionList())); + return builder.build(); + } + + Optional<String> getVariableValue(String name) { + String value = values.get(name); + if (value != null) return Optional.of(value); + if (parent != null) return parent.getVariableValue(name); + return Optional.empty(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/FormBuilder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/FormBuilder.java new file mode 100644 index 00000000000..cae5279f68a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/FormBuilder.java @@ -0,0 +1,81 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author hakonhall + */ +class FormBuilder { + /** The top-level section list in this form. */ + private final SectionList sectionList; + private final List<Section> allSections = new ArrayList<>(); + private final Map<String, VariableSection> sampleVariables = new HashMap<>(); + private final Map<String, IfSection> sampleIfSections = new HashMap<>(); + private final Map<String, ListSection> lists = new HashMap<>(); + + FormBuilder(Cursor start) { + this.sectionList = new SectionList(start, this); + } + + SectionList topLevelSectionList() { return sectionList; } + + void addLiteralSection(LiteralSection section) { + allSections.add(section); + } + + void addVariableSection(VariableSection section) { + // It's OK if the same name is used in an if-directive (as long as the value is boolean, + // determined when set on a form). + + ListSection existing = lists.get(section.name()); + if (existing != null) + throw new NameAlreadyExistsTemplateException(section.name(), existing.nameOffset(), + section.nameOffset()); + + sampleVariables.put(section.name(), section); + allSections.add(section); + } + + void addIfSection(IfSection section) { + // It's OK if the same name is used in a variable section (as long as the value is boolean, + // determined when set on a form). + + ListSection list = lists.get(section.name()); + if (list != null) + throw new NameAlreadyExistsTemplateException(section.name(), list.nameOffset(), + section.nameOffset()); + + sampleIfSections.put(section.name(), section); + allSections.add(section); + } + + void addListSection(ListSection section) { + VariableSection variableSection = sampleVariables.get(section.name()); + if (variableSection != null) + throw new NameAlreadyExistsTemplateException(section.name(), variableSection.nameOffset(), + section.nameOffset()); + + IfSection ifSection = sampleIfSections.get(section.name()); + if (ifSection != null) + throw new NameAlreadyExistsTemplateException(section.name(), ifSection.nameOffset(), + section.nameOffset()); + + ListSection previous = lists.put(section.name(), section); + if (previous != null) + throw new NameAlreadyExistsTemplateException(section.name(), previous.nameOffset(), + section.nameOffset()); + allSections.add(section); + } + + Form build() { + var form = new Form(sectionList.range(), sectionList.sections(), lists); + allSections.forEach(section -> section.setForm(form)); + return form; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java new file mode 100644 index 00000000000..8775e764b4f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java @@ -0,0 +1,68 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.Optional; + +/** + * @author hakonhall + */ +class IfSection extends Section { + private final boolean negated; + private final String name; + private final Cursor nameOffset; + private final SectionList ifSections; + private final Optional<SectionList> elseSections; + + IfSection(CursorRange range, boolean negated, String name, Cursor nameOffset, + SectionList ifSections, Optional<SectionList> elseSections) { + super(range); + this.negated = negated; + this.name = name; + this.nameOffset = nameOffset; + this.ifSections = ifSections; + this.elseSections = elseSections; + } + + String name() { return name; } + Cursor nameOffset() { return nameOffset; } + + @Override + void appendTo(StringBuilder buffer) { + Optional<String> stringValue = form().getVariableValue(name); + if (stringValue.isEmpty()) + throw new TemplateNameNotSetException(name, nameOffset); + + final boolean value; + if (stringValue.get().equals("true")) { + value = true; + } else if (stringValue.get().equals("false")) { + value = false; + } else { + throw new NotBooleanValueTemplateException(name); + } + + boolean condition = negated ? !value : value; + if (condition) { + ifSections.sections().forEach(section -> section.appendTo(buffer)); + } else if (elseSections.isPresent()) { + elseSections.get().sections().forEach(section -> section.appendTo(buffer)); + } + } + + @Override + void appendCopyTo(SectionList sectionList) { + SectionList ifSectionCopy = new SectionList(ifSections.range().start(), sectionList.formBuilder()); + ifSections.sections().forEach(section -> section.appendCopyTo(ifSectionCopy)); + + Optional<SectionList> elseSectionCopy = elseSections.map(elseSections2 -> { + SectionList elseSectionCopy2 = new SectionList(elseSections2.range().start(), sectionList.formBuilder()); + elseSections2.sections().forEach(section -> section.appendCopyTo(elseSectionCopy2)); + return elseSectionCopy2; + }); + + sectionList.appendIfSection(negated, name, nameOffset, range().end(), ifSectionCopy, elseSectionCopy); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java new file mode 100644 index 00000000000..bc68cf96153 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author hakonhall + */ +class ListSection extends Section { + private final String name; + private final Cursor nameOffset; + private final Form body; + private final List<Form> elements = new ArrayList<>(); + + ListSection(CursorRange range, String name, Cursor nameOffset, Form body) { + super(range); + this.name = name; + this.nameOffset = new Cursor(nameOffset); + this.body = body; + } + + String name() { return name; } + Cursor nameOffset() { return new Cursor(nameOffset); } + + @Override + void setForm(Form form) { + super.setForm(form); + body.setParent(form); + } + + Form add() { + Form element = body.copy(); + element.setParent(form()); + elements.add(element); + return element; + } + + @Override + void appendTo(StringBuilder buffer) { + elements.forEach(form -> form.appendTo(buffer)); + } + + @Override + void appendCopyTo(SectionList sectionList) { + // avoid copying elements for now + // Optimization: Reuse body in copy, since it is only used for copying. + + sectionList.appendListSection(name, nameOffset, range().end(), body); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java new file mode 100644 index 00000000000..c03653253af --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * Represents a template literal section + * + * @see Template + * @author hakonhall + */ +class LiteralSection extends Section { + LiteralSection(CursorRange range) { + super(range); + } + + @Override + void appendTo(StringBuilder buffer) { + range().appendTo(buffer); + } + + @Override + void appendCopyTo(SectionList sectionList) { + sectionList.appendLiteralSection(range().end()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java new file mode 100644 index 00000000000..dd92af14609 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * @author hakonhall + */ +public class NameAlreadyExistsTemplateException extends TemplateException { + public NameAlreadyExistsTemplateException(String name, CursorRange range) { + super("Name '" + name + "' already exists in the " + describeSection(range)); + } + + public NameAlreadyExistsTemplateException(String name, Cursor firstNameLocation, + Cursor secondNameLocation) { + super("Section named '" + name + "' at " + + firstNameLocation.calculateLocation().lineAndColumnText() + + " conflicts with earlier section with the same name at " + + secondNameLocation.calculateLocation().lineAndColumnText()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java new file mode 100644 index 00000000000..706d347d39d --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * @author hakonhall + */ +public class NoSuchNameTemplateException extends TemplateException { + public NoSuchNameTemplateException(CursorRange range, String name) { + super("No such element '" + name + "' in the " + describeSection(range)); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java new file mode 100644 index 00000000000..6c6d157bb47 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +/** + * @author hakonhall + */ +public class NotBooleanValueTemplateException extends TemplateException { + public NotBooleanValueTemplateException(String name) { + super(name + " was set to a non-boolean value: must be true or false"); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java new file mode 100644 index 00000000000..234915770f8 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * A section of a template text. + * + * @see Template + * @author hakonhall + */ +abstract class Section { + private final CursorRange range; + private Form form; + + protected Section(CursorRange range) { + this.range = range; + } + + void setForm(Form form) { this.form = form; } + + /** Guaranteed to return non-null after FormBuilder::build() returns. */ + protected Form form() { return form; } + + protected CursorRange range() { return range; } + + abstract void appendTo(StringBuilder buffer); + + abstract void appendCopyTo(SectionList sectionList); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java new file mode 100644 index 00000000000..b9a8c4e8c41 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java @@ -0,0 +1,68 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * A mutable list of sections at the same level that can be used to build a form, e.g. the if-body. + * + * @author hakonhall + */ +class SectionList { + private final Cursor start; + private final Cursor end; + private final FormBuilder formBuilder; + + private final List<Section> sections = new ArrayList<>(); + + SectionList(Cursor start, FormBuilder formBuilder) { + this.start = new Cursor(start); + this.end = new Cursor(start); + this.formBuilder = formBuilder; + } + + CursorRange range() { return new CursorRange(start, end); } + FormBuilder formBuilder() { return formBuilder; } + List<Section> sections() { return List.copyOf(sections); } + + void appendLiteralSection(Cursor end) { + CursorRange range = verifyAndUpdateEnd(end); + var section = new LiteralSection(range); + formBuilder.addLiteralSection(section); + sections.add(section); + } + + VariableSection appendVariableSection(String name, Cursor nameOffset, Cursor end) { + CursorRange range = verifyAndUpdateEnd(end); + var section = new VariableSection(range, name, nameOffset); + formBuilder.addVariableSection(section); + sections.add(section); + return section; + } + + void appendIfSection(boolean negated, String name, Cursor nameOffset, Cursor end, + SectionList ifSections, Optional<SectionList> elseSections) { + CursorRange range = verifyAndUpdateEnd(end); + var section = new IfSection(range, negated, name, nameOffset, ifSections, elseSections); + formBuilder.addIfSection(section); + sections.add(section); + } + + void appendListSection(String name, Cursor nameOffset, Cursor end, Form body) { + CursorRange range = verifyAndUpdateEnd(end); + var section = new ListSection(range, name, nameOffset, body); + formBuilder.addListSection(section); + sections.add(section); + } + + private CursorRange verifyAndUpdateEnd(Cursor newEnd) { + var range = new CursorRange(this.end, newEnd); + this.end.set(newEnd); + return range; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java new file mode 100644 index 00000000000..344424c7946 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +/** + * The Java representation of a template text. + * + * <p>A template is a sequence of literal text and dynamic sections defined by %{...} directives:</p> + * + * <pre> + * template: section* + * section: literal | variable | list + * literal: plain text not containing %{ + * variable: %{=id} + * if: %{if [!]id}template[%{else}template]%{end} + * list: %{list id}template%{end} + * id: a valid Java identifier + * </pre> + * + * <p>If the directive's end delimiter (}) is preceded by a "-" char, then any newline (\n) + * immediately following the end delimiter is removed.</p> + * + * <p>To use the template create a form ({@link #newForm()}), fill the form (e.g. + * {@link Form#set(String, String) Form.set()}), and render the String ({@link Form#render()}).</p> + * + * @see Form + * @see TemplateFile + * @author hakonhall + */ +public class Template { + private final Form form; + + public static Template from(String text) { return from(text, new TemplateDescriptor()); } + + public static Template from(String text, TemplateDescriptor descriptor) { + return TemplateParser.parse(text, descriptor).template(); + } + + Template(Form form) { + this.form = form; + } + + public Form newForm() { return form.copy(); } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java new file mode 100644 index 00000000000..05d4f82d8d3 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +/** + * Specifies the how to interpret a template text. + * + * @author hakonhall + */ +public class TemplateDescriptor { + + private String startDelimiter = "%{"; + private String endDelimiter = "}"; + + public TemplateDescriptor() {} + + public TemplateDescriptor(TemplateDescriptor that) { + this.startDelimiter = that.startDelimiter; + this.endDelimiter = that.endDelimiter; + } + + /** Use these delimiters instead of the standard "%{" and "}" to start and end a template directive. */ + public TemplateDescriptor setDelimiters(String startDelimiter, String endDelimiter) { + this.startDelimiter = Token.verifyDelimiter(startDelimiter); + this.endDelimiter = Token.verifyDelimiter(endDelimiter); + return this; + } + + public String startDelimiter() { return startDelimiter; } + public String endDelimiter() { return endDelimiter; } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java new file mode 100644 index 00000000000..f231583de52 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * @author hakonhall + */ +public class TemplateException extends RuntimeException { + public TemplateException(String message) { super(message); } + + protected static String describeSection(CursorRange range) { + var startLocation = range.start().calculateLocation(); + var endLocation = range.end().calculateLocation(); + return "template section starting at line " + startLocation.line() + " and column " + startLocation.column() + + ", and ending at line " + endLocation.line() + " and column " + endLocation.column(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateFile.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateFile.java new file mode 100644 index 00000000000..0c1a26f4f65 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateFile.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +import java.nio.file.Path; + +/** + * Parses a template file, see {@link Template} for details. + * + * @author hakonhall + */ +public class TemplateFile { + public static Template read(Path path) { return read(path, new TemplateDescriptor()); } + + public static Template read(Path path, TemplateDescriptor descriptor) { + String content = new UnixPath(path).readUtf8File(); + return Template.from(content, descriptor); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java new file mode 100644 index 00000000000..d65c2a7c4d6 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; + +/** + * @author hakonhall + */ +public class TemplateNameNotSetException extends TemplateException { + public TemplateNameNotSetException(String name, Cursor nameOffset) { + super("Variable at " + nameOffset.calculateLocation().lineAndColumnText() + " has not been set: " + name); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java new file mode 100644 index 00000000000..93a83a3d29f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java @@ -0,0 +1,161 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; + +import java.util.EnumSet; +import java.util.Optional; + +/** + * Parses a template String, see {@link Template} for details. + * + * @author hakonhall + */ +class TemplateParser { + private final TemplateDescriptor descriptor; + private final Cursor start; + private final Cursor current; + private final FormBuilder formBuilder; + + static TemplateParser parse(String text, TemplateDescriptor descriptor) { + return parse(new TemplateDescriptor(descriptor), new Cursor(text), EnumSet.of(Sentinel.EOT)); + } + + private static TemplateParser parse(TemplateDescriptor descriptor, Cursor start, EnumSet<Sentinel> sentinel) { + var parser = new TemplateParser(descriptor, start); + parser.parse(parser.formBuilder.topLevelSectionList(), sentinel); + return parser; + } + + private enum Sentinel { ELSE, END, EOT } + + private TemplateParser(TemplateDescriptor descriptor, Cursor start) { + this.descriptor = descriptor; + this.start = new Cursor(start); + this.current = new Cursor(start); + this.formBuilder = new FormBuilder(start); + } + + Template template() { return new Template(formBuilder.build()); } + + private Sentinel parse(SectionList sectionList, EnumSet<Sentinel> sentinels) { + do { + current.advanceTo(descriptor.startDelimiter()); + if (!current.equals(start)) { + sectionList.appendLiteralSection(current); + } + + if (current.eot()) { + if (!sentinels.contains(Sentinel.EOT)) { + throw new BadTemplateException(current, + "Missing end directive for section started at " + + start.calculateLocation().lineAndColumnText()); + } + return Sentinel.EOT; + } + + Optional<Sentinel> sentinel = parseSection(sectionList, sentinels); + if (sentinel.isPresent()) return sentinel.get(); + } while (true); + } + + private Optional<Sentinel> parseSection(SectionList sectionList, EnumSet<Sentinel> sentinels) { + current.skip(descriptor.startDelimiter()); + + if (current.skip(Token.VARIABLE_DIRECTIVE_CHAR)) { + parseVariableSection(sectionList); + } else { + var startOfType = new Cursor(current); + String type = skipId().orElseThrow(() -> new BadTemplateException(current, "Missing section name")); + + switch (type) { + case "else": + if (!sentinels.contains(Sentinel.ELSE)) + throw new BadTemplateException(startOfType, "Extraneous 'else'"); + parseEndDirective(); + return Optional.of(Sentinel.ELSE); + case "end": + if (!sentinels.contains(Sentinel.END)) + throw new BadTemplateException(startOfType, "Extraneous 'end'"); + parseEndDirective(); + return Optional.of(Sentinel.END); + case "if": + parseIfSection(sectionList); + break; + case "list": + parseListSection(sectionList); + break; + default: + throw new BadTemplateException(startOfType, "Unknown section '" + type + "'"); + } + } + + return Optional.empty(); + } + + private void parseVariableSection(SectionList sectionList) { + var nameStart = new Cursor(current); + String name = parseId(); + parseEndDelimiter(true); + sectionList.appendVariableSection(name, nameStart, current); + } + + private void parseEndDirective() { + parseEndDelimiter(true); + } + + private void parseListSection(SectionList sectionList) { + skipRequiredWhitespaces(); + var startOfName = new Cursor(current); + String name = parseId(); + parseEndDelimiter(true); + + TemplateParser bodyParser = parse(descriptor, current, EnumSet.of(Sentinel.END)); + current.set(bodyParser.current); + + sectionList.appendListSection(name, startOfName, current, bodyParser.formBuilder.build()); + } + + private void parseIfSection(SectionList sectionList) { + skipRequiredWhitespaces(); + boolean negated = current.skip(Token.NEGATE_CHAR); + current.skipWhitespaces(); + var startOfName = new Cursor(current); + String name = parseId(); + parseEndDelimiter(true); + + SectionList ifSectionList = new SectionList(current, formBuilder); + Sentinel ifSentinel = parse(ifSectionList, EnumSet.of(Sentinel.ELSE, Sentinel.END)); + + Optional<SectionList> elseSectionList = Optional.empty(); + if (ifSentinel == Sentinel.ELSE) { + elseSectionList = Optional.of(new SectionList(current, formBuilder)); + parse(elseSectionList.get(), EnumSet.of(Sentinel.END)); + } + + sectionList.appendIfSection(negated, name, startOfName, current, ifSectionList, elseSectionList); + } + + private void skipRequiredWhitespaces() { + if (!current.skipWhitespaces()) { + throw new BadTemplateException(current, "Expected whitespace"); + } + } + + private String parseId() { + return skipId().orElseThrow(() -> new BadTemplateException(current, "Expected identifier")); + } + + private Optional<String> skipId() { return Token.skipId(current); } + + private boolean parseEndDelimiter(boolean skipNewline) { + boolean removeNewline = current.skip(Token.REMOVE_NEWLINE_CHAR); + if (!current.skip(descriptor.endDelimiter())) + throw new BadTemplateException(current, "Expected section end (" + descriptor.endDelimiter() + ")"); + + if (skipNewline && removeNewline) + current.skip('\n'); + + return removeNewline; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java new file mode 100644 index 00000000000..a83dab72025 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.Optional; + +/** + * @author hakonhall + */ +class Token { + static final char NEGATE_CHAR = '!'; + static final char REMOVE_NEWLINE_CHAR = '-'; + static final char VARIABLE_DIRECTIVE_CHAR = '='; + + static Optional<String> skipId(Cursor cursor) { + if (cursor.eot() || !isIdStart(cursor.getChar())) return Optional.empty(); + + Cursor start = new Cursor(cursor); + cursor.increment(); + + while (!cursor.eot() && isIdPart(cursor.getChar())) + cursor.increment(); + + return Optional.of(new CursorRange(start, cursor).string()); + } + + /** A delimiter either starts a directive (e.g. %{) or ends it (e.g. }). */ + static String verifyDelimiter(String delimiter) { + if (!isAsciiToken(delimiter)) { + throw new IllegalArgumentException("Invalid delimiter: '" + delimiter + "'"); + } + return delimiter; + } + + /** Returns true for a non-empty string with only ASCII token characters. */ + private static boolean isAsciiToken(String string) { + if (string.isEmpty()) return false; + for (char c : string.toCharArray()) { + if (!isAsciiTokenChar(c)) return false; + } + return true; + } + + /** Returns true if char is a printable ASCII character except space (isgraph(3)). */ + private static boolean isAsciiTokenChar(char c) { + // 0x1F unit separator + // 0x20 space + // 0x21 ! + // ... + // 0x7E ~ + // 0x7F del + return 0x20 < c && c < 0x7F; + } + + // Our identifiers are equivalent to a Java identifiers. + private static boolean isIdStart(char c) { return Character.isJavaIdentifierStart(c); } + private static boolean isIdPart(char c) { return Character.isJavaIdentifierPart(c); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java new file mode 100644 index 00000000000..bf211a01190 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * Represents a template variable section + * + * @see Template + * @author hakonhall + */ +class VariableSection extends Section { + private final String name; + private final Cursor nameOffset; + + VariableSection(CursorRange range, String name, Cursor nameOffset) { + super(range); + this.name = name; + this.nameOffset = nameOffset; + } + + String name() { return name; } + Cursor nameOffset() { return new Cursor(nameOffset); } + + @Override + void appendTo(StringBuilder buffer) { + String value = form().getVariableValue(name) + .orElseThrow(() -> new TemplateNameNotSetException(name, nameOffset)); + buffer.append(value); + } + + @Override + void appendCopyTo(SectionList sectionList) { + sectionList.appendVariableSection(name, nameOffset, range().end()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java new file mode 100644 index 00000000000..5bb8f656305 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java new file mode 100644 index 00000000000..2fc3f8bac60 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java @@ -0,0 +1,165 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.text; + +import java.util.Objects; + +/** + * Cursor is a mutable offset into a fixed String, and useful for String parsing. + * + * @author hakonhall + */ +// @Mutable +public class Cursor { + private final String text; + private int offset; + private TextLocation locationCache; + + /** Creates a pointer to the first char of {@code text}, which is EOT if {@code text} is empty. */ + public Cursor(String text) { this(text, 0, new TextLocation()); } + + public Cursor(Cursor that) { this(that.text, that.offset, that.locationCache); } + + private Cursor(String text, int offset, TextLocation location) { + this.text = Objects.requireNonNull(text); + this.offset = offset; + this.locationCache = Objects.requireNonNull(location); + } + + /** Returns the substring of {@code text} starting at {@link #offset()} (to EOT). */ + @Override + public String toString() { return text.substring(offset); } + + public String fullText() { return text; } + public int offset() { return offset; } + public boolean bot() { return offset == 0; } + public boolean eot() { return offset == text.length(); } + public boolean startsWith(char c) { return offset < text.length() && text.charAt(offset) == c; } + public boolean startsWith(String prefix) { return text.startsWith(prefix, offset); } + + /** @throws IndexOutOfBoundsException if {@link #eot()}. */ + public char getChar() { return text.charAt(offset); } + + /** The number of chars between pointer and EOT. */ + public int length() { return text.length() - offset; } + + /** Calculate the current text location in O(length(text)). */ + public TextLocation calculateLocation() { + if (offset < locationCache.offset()) { + locationCache = new TextLocation(); + } else if (offset == locationCache.offset()) { + return locationCache; + } + + int lineIndex = locationCache.lineIndex(); + int columnIndex = locationCache.columnIndex(); + for (int i = locationCache.offset(); i < offset; ++i) { + if (text.charAt(i) == '\n') { + ++lineIndex; + columnIndex = 0; + } else { + ++columnIndex; + } + } + + locationCache = new TextLocation(offset, lineIndex, columnIndex); + return locationCache; + } + + public void set(Cursor that) { + if (that.text != text) { + throw new IllegalArgumentException("'that' doesn't refer to the same text"); + } + + this.offset = that.offset; + } + + /** Advance substring.length() if this startsWith the substring, returning true if so. */ + public boolean skip(String substring) { + if (startsWith(substring)) { + offset += substring.length(); + return true; + } else { + return false; + } + } + + public boolean skip(char c) { + if (startsWith(c)) { + ++offset; + return true; + } else { + return false; + } + } + + /** If the current char is a whitespace, skip it and return true. */ + public boolean skipWhitespace() { + if (!eot() && Character.isWhitespace(getChar())) { + ++offset; + return true; + } else { + return false; + } + } + + /** Returns true if at least one whitespace was skipped. */ + public boolean skipWhitespaces() { + if (skipWhitespace()) { + while (skipWhitespace()) + ++offset; + return true; + } else { + return false; + } + } + + /** Return false if eot(), otherwise advance to the next char and return true. */ + public boolean increment() { + if (eot()) return false; + ++offset; + return true; + } + + /** + * Advance {@code distance} chars until bot() or eot() is reached (distance may be negative), + * and return true if this cursor moved the full distance. + */ + public boolean advance(int distance) { + int newOffset = offset + distance; + if (newOffset < 0) { + this.offset = 0; + return false; + } else if (newOffset > text.length()) { + this.offset = text.length(); + return false; + } else { + this.offset = newOffset; + return true; + } + } + + /** Advance pointer until start of needle is found (and return true), or EOT is reached (and return false). */ + public boolean advanceTo(String needle) { + int index = text.indexOf(needle, offset); + if (index == -1) { + offset = text.length(); + return false; // and eot() is true + } else { + offset = index; + return true; // and eot() is false + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Cursor cursor = (Cursor) o; + return offset == cursor.offset && text.equals(cursor.text); + } + + @Override + public int hashCode() { + return Objects.hash(text, offset); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java new file mode 100644 index 00000000000..23ac69ccee2 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.text; + +/** + * A start- and end- offset in an underlying String. + * + * @author hakonhall + */ +public class CursorRange { + private final Cursor start; + private final Cursor end; + + @SuppressWarnings("StringEquality") + public CursorRange(Cursor start, Cursor end) { + if (start.fullText() != end.fullText()) { + throw new IllegalArgumentException("start and end points to different texts"); + } + + if (start.offset() > end.offset()) { + throw new IllegalArgumentException("start offset " + start.offset() + + " is beyond end offset " + end.offset()); + } + + this.start = new Cursor(start); + this.end = new Cursor(end); + } + + public CursorRange(CursorRange that) { + this.start = new Cursor(that.start); + this.end = new Cursor(that.end); + } + + public Cursor start() { return new Cursor(start); } + public Cursor end() { return new Cursor(end); } + public int length() { return end.offset() - start.offset(); } + public String string() { return start.fullText().substring(start.offset(), end.offset()); } + public void appendTo(StringBuilder buffer) { buffer.append(start.fullText(), start.offset(), end.offset()); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java new file mode 100644 index 00000000000..32441c842b0 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.text; + +/** + * The location within an implied multi-line String. + * + * @author hakonhall + */ +//@Immutable +public class TextLocation { + private final int offset; + private final int lineIndex; + private final int columnIndex; + + public TextLocation() { this(0, 0, 0); } + + public TextLocation(int offset, int lineIndex, int columnIndex) { + this.offset = offset; + this.lineIndex = lineIndex; + this.columnIndex = columnIndex; + } + + public int offset() { return offset; } + public int lineIndex() { return lineIndex; } + public int line() { return lineIndex + 1; } + public int columnIndex() { return columnIndex; } + public int column() { return columnIndex + 1; } + + public String lineAndColumnText() { return "line " + line() + " and column " + column(); } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateFileTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateFileTest.java new file mode 100644 index 00000000000..40913184a67 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateFileTest.java @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author hakonhall + */ +class TemplateFileTest { + @Test + void verifyVariableSection() { + Form form = getForm("template1.tmp"); + form.set("varname", "varvalue"); + assertEquals("variable section 'varvalue'\n" + + "end of text\n", form.render()); + } + + @Test + void verifySimpleListSection() { + Form form = getForm("template1.tmp"); + form.set("varname", "varvalue") + .add("listname") + .set("varname", "different varvalue") + .set("varname2", "varvalue2"); + assertEquals("variable section 'varvalue'\n" + + "same variable section 'different varvalue'\n" + + "different variable section 'varvalue2'\n" + + "between ends\n" + + "end of text\n", form.render()); + } + + @Test + void verifyNestedListSection() { + Form form = getForm("template2.tmp"); + Form A0 = form.add("listA"); + Form A0B0 = A0.add("listB"); + Form A0B1 = A0.add("listB"); + + Form A1 = form.add("listA"); + Form A1B0 = A1.add("listB"); + assertEquals("body A\n" + + "body B\n" + + "body B\n" + + "body A\n" + + "body B\n", + form.render()); + } + + @Test + void verifyVariableReferences() { + Form form = getForm("template3.tmp"); + form.set("varname", "varvalue") + .set("innerVarSetAtTop", "val2"); + form.add("l"); + form.add("l") + .set("varname", "varvalue2"); + assertEquals("varvalue\n" + + "varvalue\n" + + "inner varvalue\n" + + "val2\n" + + "inner varvalue2\n" + + "val2\n", + form.render()); + } + + private Form getForm(String filename) { + return TemplateFile.read(Path.of("src/test/resources/" + filename)).newForm(); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java new file mode 100644 index 00000000000..8d503dd4784 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author hakonhall + */ +public class TemplateTest { + @Test + void verifyNewlineRemoval() { + Form form = makeForm("a%{list a}\n" + + "b%{end}\n" + + "c%{list c-}\n" + + "d%{end-}\n" + + "e\n"); + form.add("a"); + form.add("c"); + + assertEquals("a\n" + + "b\n" + + "cde\n", + form.render()); + } + + @Test + void verifyIfSection() { + Template template = Template.from("Hello%{if cond} world%{end}!"); + assertEquals("Hello world!", template.newForm().set("cond", true).render()); + assertEquals("Hello!", template.newForm().set("cond", false).render()); + } + + @Test + void verifyComplexIfSection() { + Template template = Template.from("%{if cond-}\n" + + "var: %{=varname}\n" + + "if: %{if !inner}inner is false%{end}\n" + + "list: %{list formname}element%{end}\n" + + "%{end-}\n"); + + assertEquals("", template.newForm().set("cond", false).render()); + + assertEquals("var: varvalue\n" + + "if: \n" + + "list: \n", + template.newForm() + .set("cond", true) + .set("varname", "varvalue") + .set("inner", true) + .render()); + + Form form = template.newForm() + .set("cond", true) + .set("varname", "varvalue") + .set("inner", false); + form.add("formname"); + + assertEquals("var: varvalue\n" + + "if: inner is false\n" + + "list: element\n", form.render()); + } + + @Test + void verifyElse() { + var template = Template.from("%{if cond-}\n" + + "if body\n" + + "%{else-}\n" + + "else body\n" + + "%{end-}\n"); + assertEquals("if body\n", template.newForm().set("cond", true).render()); + assertEquals("else body\n", template.newForm().set("cond", false).render()); + } + + private Form makeForm(String templateText) { + return Template.from(templateText).newForm(); + } +} diff --git a/node-admin/src/test/resources/template1.tmp b/node-admin/src/test/resources/template1.tmp new file mode 100644 index 00000000000..3468709cc2e --- /dev/null +++ b/node-admin/src/test/resources/template1.tmp @@ -0,0 +1,10 @@ +variable section '%{=varname}' +%{list listname-} +same variable section '%{=varname}' +different variable section '%{=varname2}' +%{list innerlistname-} +inner form text +%{end-} +between ends +%{end-} +end of text diff --git a/node-admin/src/test/resources/template2.tmp b/node-admin/src/test/resources/template2.tmp new file mode 100644 index 00000000000..3bfa30ec7d3 --- /dev/null +++ b/node-admin/src/test/resources/template2.tmp @@ -0,0 +1,4 @@ +%{list listA-}body A +%{list listB-}body B +%{end-} +%{end-} diff --git a/node-admin/src/test/resources/template3.tmp b/node-admin/src/test/resources/template3.tmp new file mode 100644 index 00000000000..454b01761b5 --- /dev/null +++ b/node-admin/src/test/resources/template3.tmp @@ -0,0 +1,6 @@ +%{=varname} +%{=varname} +%{list l-} +inner %{=varname} +%{=innerVarSetAtTop} +%{end-} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 80c192f8353..abacaf8e264 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -72,7 +72,8 @@ public class Autoscaler { if (scaledIn(clusterModel.scalingDuration(), cluster)) return Advice.dontScale(Status.waiting, - "Won't autoscale now: Less than " + clusterModel.scalingDuration() + " since last resource change"); + "Won't autoscale now: Less than " + clusterModel.scalingDuration() + + " since last resource change"); if (clusterModel.nodeTimeseries().measurementsPerNode() < minimumMeasurementsPerNode(clusterModel.scalingDuration())) return Advice.none(Status.waiting, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 874e9cbe2c5..ca14a1be4c4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -369,9 +369,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { static Map<String, String> dimensions(ApplicationId application, ClusterSpec.Id cluster) { Map<String, String> dimensions = new HashMap<>(dimensions(application)); - //TODO: Remove "clusterId" once internal aggregation uses "clusterid" dimensions.put("clusterid", cluster.value()); - dimensions.put("clusterId", cluster.value()); return dimensions; } diff --git a/parent/pom.xml b/parent/pom.xml index 7ab4e2ba57e..d1da2a51a98 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -481,7 +481,7 @@ <dependency> <groupId>com.ibm.icu</groupId> <artifactId>icu4j</artifactId> - <version>57.1</version> + <version>57.2</version> </dependency> <dependency> <groupId>com.infradna.tool</groupId> diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index 5ffb37feb70..d1a56b8ac8a 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -121,6 +121,12 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { return getInstallVariable("zts_url"); } + @Override + public String zooKeeperSnapshotMethod() { + String vespaZookeeperSnapshotMethod = System.getenv("VESPA_ZOOKEEPER_SNAPSHOT_METHOD"); + return vespaZookeeperSnapshotMethod == null ? "" : vespaZookeeperSnapshotMethod; + } + static ConfigServer[] toConfigServers(String configserversString) { return multiValueParameterStream(configserversString) .map(CloudConfigInstallVariables::toConfigServer) diff --git a/storage/src/vespa/storage/common/dummy_mbus_messages.h b/storage/src/vespa/storage/common/dummy_mbus_messages.h new file mode 100644 index 00000000000..10ecf7b6ed7 --- /dev/null +++ b/storage/src/vespa/storage/common/dummy_mbus_messages.h @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/message.h> + +/** + * Dummy-implementation of mbus::Message and mbus::Reply to be used when interacting with + * MessageBus IThrottlePolicy subclasses, as these expect message instances as parameters. + */ + +namespace storage { + +template <typename Base> +class DummyMbusMessage : public Base { + static const mbus::string NAME; +public: + const mbus::string& getProtocol() const override { return NAME; } + uint32_t getType() const override { return 0x1badb007; } + uint8_t priority() const override { return 255; } +}; + +template <typename Base> +const mbus::string DummyMbusMessage<Base>::NAME = "FooBar"; + +class DummyMbusRequest final : public DummyMbusMessage<mbus::Message> { +public: + // getApproxSize() returns 1 by default. + // Approximate size of messages allowed by throttle policy is implicitly added to + // internal StaticThrottlePolicy pending size tracking and associated with the + // internal mbus context of the message. + // Since we have no connection between the request and reply instances used when + // interacting with the policy, we have to make sure they cancel each other out + // (i.e. += 0, -= 0). + // Not doing this would cause the StaticThrottlePolicy to keep adding a single byte + // of pending size for each message allowed by the policy. + uint32_t getApproxSize() const override { return 0; } +}; + +class DummyMbusReply final : public DummyMbusMessage<mbus::Reply> {}; + +} diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 5e0ea0359dc..2ccbc7a85ef 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -940,7 +940,10 @@ FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout) } else if (should_throttle_op && !throttle_token.valid()) { // Important: _non-blocking_ attempt at getting a throttle token. throttle_token = _owner.operation_throttler().try_acquire_one(); - was_throttled = !throttle_token.valid(); + if (!throttle_token.valid()) { + was_throttled = true; + _metrics->throttled_persistence_thread_polls.inc(); + } } if (!should_throttle_op || throttle_token.valid()) { return getMessage(guard, idx, iter, std::move(throttle_token)); @@ -956,10 +959,11 @@ FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout) // prevents RPC threads from pushing onto the queue. guard.unlock(); throttle_token = _owner.operation_throttler().blocking_acquire_one(timeout); + guard.lock(); if (!throttle_token.valid()) { + _metrics->timeouts_waiting_for_throttle_token.inc(); return {}; // Already exhausted our timeout window. } - guard.lock(); } } } @@ -984,6 +988,8 @@ FileStorHandlerImpl::Stripe::get_next_async_message(monitor_guard& guard) auto throttle_token = _owner.operation_throttler().try_acquire_one(); if (throttle_token.valid()) { return getMessage(guard, idx, iter, std::move(throttle_token)); + } else { + _metrics->throttled_rpc_direct_dispatches.inc(); } } return {}; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index a98077da57a..6cb76c32997 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -190,7 +190,16 @@ FileStorThreadMetrics::~FileStorThreadMetrics() = default; FileStorStripeMetrics::FileStorStripeMetrics(const std::string& name, const std::string& description) : MetricSet(name, {{"partofsum"}}, description), - averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this) + averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this), + throttled_rpc_direct_dispatches("throttled_rpc_direct_dispatches", {}, + "Number of times an RPC thread could not directly dispatch an async operation " + "directly to Proton because it was disallowed by the throttle policy", this), + throttled_persistence_thread_polls("throttled_persistence_thread_polls", {}, + "Number of times a persistence thread could not immediately dispatch a " + "queued async operation because it was disallowed by the throttle policy", this), + timeouts_waiting_for_throttle_token("timeouts_waiting_for_throttle_token", {}, + "Number of times a persistence thread timed out waiting for an available " + "throttle policy token", this) { } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index d8135c9aeca..1bcbacd2ca6 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -131,6 +131,9 @@ class FileStorStripeMetrics : public metrics::MetricSet public: using SP = std::shared_ptr<FileStorStripeMetrics>; metrics::DoubleAverageMetric averageQueueWaitingTime; + metrics::LongCountMetric throttled_rpc_direct_dispatches; + metrics::LongCountMetric throttled_persistence_thread_polls; + metrics::LongCountMetric timeouts_waiting_for_throttle_token; FileStorStripeMetrics(const std::string& name, const std::string& description); ~FileStorStripeMetrics() override; }; diff --git a/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp b/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp index b72b1a8ba28..7b05decb851 100644 --- a/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp +++ b/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "shared_operation_throttler.h" #include <vespa/messagebus/dynamicthrottlepolicy.h> -#include <vespa/messagebus/message.h> +#include <vespa/storage/common/dummy_mbus_messages.h> #include <condition_variable> #include <cassert> #include <mutex> @@ -28,19 +28,6 @@ private: void release_one() noexcept override { /* no-op */ } }; -// Class used to sneakily get around IThrottlePolicy only accepting MBus objects -template <typename Base> -class DummyMbusMessage final : public Base { - static const mbus::string NAME; -public: - const mbus::string& getProtocol() const override { return NAME; } - uint32_t getType() const override { return 0x1badb007; } - uint8_t priority() const override { return 255; } -}; - -template <typename Base> -const mbus::string DummyMbusMessage<Base>::NAME = "FooBar"; - class DynamicOperationThrottler final : public SharedOperationThrottler { mutable std::mutex _mutex; std::condition_variable _cond; @@ -58,6 +45,11 @@ public: uint32_t waiting_threads() const noexcept override; private: void release_one() noexcept override; + // Non-const since actually checking the send window of a dynamic throttler might change + // it if enough time has passed. + [[nodiscard]] bool has_spare_capacity_in_active_window() noexcept; + void add_one_to_active_window_size(); + void subtract_one_from_active_window_size(); }; DynamicOperationThrottler::DynamicOperationThrottler(uint32_t min_size_and_window_increment) @@ -71,20 +63,42 @@ DynamicOperationThrottler::DynamicOperationThrottler(uint32_t min_size_and_windo DynamicOperationThrottler::~DynamicOperationThrottler() = default; +bool +DynamicOperationThrottler::has_spare_capacity_in_active_window() noexcept +{ + DummyMbusRequest dummy_request; + return _throttle_policy.canSend(dummy_request, _pending_ops); +} + +void +DynamicOperationThrottler::add_one_to_active_window_size() +{ + DummyMbusRequest dummy_request; + _throttle_policy.processMessage(dummy_request); + ++_pending_ops; +} + +void +DynamicOperationThrottler::subtract_one_from_active_window_size() +{ + DummyMbusReply dummy_reply; + _throttle_policy.processReply(dummy_reply); + assert(_pending_ops > 0); + --_pending_ops; +} + DynamicOperationThrottler::Token DynamicOperationThrottler::blocking_acquire_one() noexcept { std::unique_lock lock(_mutex); - DummyMbusMessage<mbus::Message> dummy_msg; - if (!_throttle_policy.canSend(dummy_msg, _pending_ops)) { + if (!has_spare_capacity_in_active_window()) { ++_waiting_threads; _cond.wait(lock, [&] { - return _throttle_policy.canSend(dummy_msg, _pending_ops); + return has_spare_capacity_in_active_window(); }); --_waiting_threads; } - _throttle_policy.processMessage(dummy_msg); - ++_pending_ops; + add_one_to_active_window_size(); return Token(this, TokenCtorTag{}); } @@ -92,19 +106,17 @@ DynamicOperationThrottler::Token DynamicOperationThrottler::blocking_acquire_one(vespalib::duration timeout) noexcept { std::unique_lock lock(_mutex); - DummyMbusMessage<mbus::Message> dummy_msg; - if (!_throttle_policy.canSend(dummy_msg, _pending_ops)) { + if (!has_spare_capacity_in_active_window()) { ++_waiting_threads; const bool accepted = _cond.wait_for(lock, timeout, [&] { - return _throttle_policy.canSend(dummy_msg, _pending_ops); + return has_spare_capacity_in_active_window(); }); --_waiting_threads; if (!accepted) { return Token(); } } - _throttle_policy.processMessage(dummy_msg); - ++_pending_ops; + add_one_to_active_window_size(); return Token(this, TokenCtorTag{}); } @@ -112,12 +124,10 @@ DynamicOperationThrottler::Token DynamicOperationThrottler::try_acquire_one() noexcept { std::unique_lock lock(_mutex); - DummyMbusMessage<mbus::Message> dummy_msg; - if (!_throttle_policy.canSend(dummy_msg, _pending_ops)) { + if (!has_spare_capacity_in_active_window()) { return Token(); } - _throttle_policy.processMessage(dummy_msg); - ++_pending_ops; + add_one_to_active_window_size(); return Token(this, TokenCtorTag{}); } @@ -125,11 +135,9 @@ void DynamicOperationThrottler::release_one() noexcept { std::unique_lock lock(_mutex); - DummyMbusMessage<mbus::Reply> dummy_reply; - _throttle_policy.processReply(dummy_reply); - assert(_pending_ops > 0); - --_pending_ops; - if (_waiting_threads > 0) { + subtract_one_from_active_window_size(); + // Only wake up a waiting thread if doing so would possibly result in success. + if ((_waiting_threads > 0) && has_spare_capacity_in_active_window()) { lock.unlock(); _cond.notify_one(); } diff --git a/storage/src/vespa/storage/persistence/shared_operation_throttler.h b/storage/src/vespa/storage/persistence/shared_operation_throttler.h index 2e1de86c4b8..4ee8d017c05 100644 --- a/storage/src/vespa/storage/persistence/shared_operation_throttler.h +++ b/storage/src/vespa/storage/persistence/shared_operation_throttler.h @@ -60,8 +60,9 @@ public: // Exposed for unit testing only. [[nodiscard]] virtual uint32_t waiting_threads() const noexcept = 0; + // Creates a throttler that does exactly zero throttling (but also has zero overhead and locking) static std::unique_ptr<SharedOperationThrottler> make_unlimited_throttler(); - + // Creates a throttler that uses a MessageBus DynamicThrottlePolicy under the hood static std::unique_ptr<SharedOperationThrottler> make_dynamic_throttler(uint32_t min_size_and_window_increment); private: // Exclusively called from a valid Token. Thread safe. diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index bc2f54e5a50..2a30acb1a74 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -2,6 +2,7 @@ #include "mergethrottler.h" #include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/common/dummy_mbus_messages.h> #include <vespa/storage/persistence/messages.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/messagebus/message.h> @@ -27,22 +28,6 @@ struct NodeComparator { } }; -// Class used to sneakily get around IThrottlePolicy only accepting -// messagebus objects -template <typename Base> -class DummyMbusMessage : public Base { -private: - static const mbus::string NAME; -public: - const mbus::string& getProtocol() const override { return NAME; } - uint32_t getType() const override { return 0x1badb007; } - - uint8_t priority() const override { return 255; } -}; - -template <typename Base> -const mbus::string DummyMbusMessage<Base>::NAME = "SkyNet"; - } MergeThrottler::ChainedMergeState::ChainedMergeState() @@ -310,7 +295,7 @@ MergeThrottler::onFlush(bool /*downwards*/) "own the command", merge.first.toString().c_str()); } - DummyMbusMessage<mbus::Reply> dummyReply; + DummyMbusReply dummyReply; _throttlePolicy->processReply(dummyReply); } for (auto& entry : _queue) { @@ -419,7 +404,7 @@ MergeThrottler::enqueue_merge_for_later_processing( bool MergeThrottler::canProcessNewMerge() const { - DummyMbusMessage<mbus::Message> dummyMsg; + DummyMbusRequest dummyMsg; return _throttlePolicy->canSend(dummyMsg, _merges.size()); } @@ -858,7 +843,7 @@ MergeThrottler::processNewMergeCommand( LOG(debug, "Added merge %s to internal state", mergeCmd.toString().c_str()); - DummyMbusMessage<mbus::Message> dummyMsg; + DummyMbusRequest dummyMsg; _throttlePolicy->processMessage(dummyMsg); bool execute = false; @@ -1058,7 +1043,7 @@ MergeThrottler::processMergeReply( updateOperationMetrics(mergeReply.getResult(), _metrics->local); } - DummyMbusMessage<mbus::Reply> dummyReply; + DummyMbusReply dummyReply; if (mergeReply.getResult().failed()) { // Must be sure to add an error if reply contained a failure, since // DynamicThrottlePolicy penalizes on failed transmissions diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 9f27ac507c9..da22dbdc336 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -113,7 +113,9 @@ public abstract class Maintainer implements Runnable { successFactor = maintain(); } catch (UncheckedTimeoutException e) { - if ( ! ignoreCollision) + if (ignoreCollision) + successFactor = 1; + else log.log(Level.WARNING, this + " collided with another run. Will retry in " + interval); } catch (Throwable e) { diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index 5157dd5d59c..8b22f658a94 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -41,6 +41,7 @@ public class Configurator { System.setProperty("zookeeper.authProvider.x509", "com.yahoo.vespa.zookeeper.VespaMtlsAuthenticationProvider"); // Need to set this as a system property, otherwise it will be parsed for _every_ packet and an exception will be thrown (and handled) System.setProperty("zookeeper.globalOutstandingLimit", "1000"); + System.setProperty("zookeeper.snapshot.compression.method", zookeeperServerConfig.snapshotMethod()); } void writeConfigToDisk() { writeConfigToDisk(VespaTlsConfig.fromSystem()); } |