diff options
62 files changed, 542 insertions, 416 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java index bea7c95d412..112385e1801 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java @@ -9,12 +9,12 @@ import com.yahoo.documentmodel.DocumentTypeRepo; * It contains a search manager managing all specified search definitions. * It contains a storage manager managing all specified storage definitions. * - * @author baldersheim - * @since 2010-02-19 + * @author baldersheim */ public class DocumentModel { - private DocumentTypeRepo documentMan = new DocumentTypeRepo(); - private SearchManager searchMan = new SearchManager(); + + private final DocumentTypeRepo documentMan = new DocumentTypeRepo(); + private final SearchManager searchMan = new SearchManager(); /** * @@ -26,6 +26,6 @@ public class DocumentModel { * * @return Returns the SearchManager */ - public SearchManager getSearchManager() { return searchMan; } + public SearchManager getSearchManager() { return searchMan; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java index d4b4a747717..5e10001baf6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java @@ -21,7 +21,7 @@ public class DefaultMetrics { public static final String defaultMetricSetId = "default"; - public static MetricSet defaultMetricSet = createMetricSet(); + public static final MetricSet defaultMetricSet = createMetricSet(); private static MetricSet createMetricSet() { return new MetricSet(defaultMetricSetId, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 302e8eff2d8..413c6f07efd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -83,8 +83,6 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.zookeeperClientPort().isPresent()) { builder.clientPort(options.zookeeperClientPort().get()); } - - builder.snapshotMethod(options.zooKeeperSnapshotMethod()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index c61c140c05b..5e7c22f8844 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -21,13 +21,11 @@ public interface CloudConfigOptions { Optional<Integer> rpcPort(); Optional<Boolean> multiTenant(); Optional<Boolean> hostedVespa(); - ConfigServer[] allConfigServers(); int[] configServerZookeeperIds(); Optional<Integer> zookeeperClientPort(); String[] configModelPluginDirs(); Optional<Long> sessionLifeTimeSecs(); - Optional<Long> zookeeperBarrierTimeout(); //in seconds Optional<Integer> zookeeperElectionPort(); Optional<Integer> zookeeperQuorumPort(); @@ -38,5 +36,4 @@ public interface CloudConfigOptions { Optional<String> loadBalancerAddress(); Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); - String zooKeeperSnapshotMethod(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 4ca85a19c35..29cff23d3bd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -73,7 +73,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); assertEquals(2, config.myid()); - assertEquals("", config.snapshotMethod()); + assertEquals("gz", config.snapshotMethod()); } @Test(expected = IllegalArgumentException.class) @@ -152,8 +152,7 @@ public class ConfigserverClusterTest { .useVespaVersionInRequest(true) .hostedVespa(hostedVespa) .environment("test") - .region("bar") - .zooKeeperSnapshotMethod(hostedVespa ? "gz" : ""); + .region("bar"); Optional.of(configServerHostnames) .filter(hostnames -> !hostnames.isEmpty()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index fc7f8674149..95c3262f2f5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; -import java.util.Objects; import java.util.Optional; /** @@ -18,7 +17,6 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); - private String zooKeeperSnapshotMethod = ""; @Override public Optional<Integer> rpcPort() { @@ -108,9 +106,6 @@ public class TestOptions implements CloudConfigOptions { return Optional.empty(); } - @Override - public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } - public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -136,10 +131,4 @@ public class TestOptions implements CloudConfigOptions { return this; } - public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { - Objects.requireNonNull(snapshotMethod); - this.zooKeeperSnapshotMethod = snapshotMethod; - return this; - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 57e49ef3e8d..8d51a91bc3a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.config.server.version.VersionState; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.yolean.Exceptions; +import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -73,6 +74,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; private final ExecutorService rpcServerExecutor; private final ConfigServerMaintenance configServerMaintenance; + private final Clock clock; @SuppressWarnings("unused") // Injected component @Inject @@ -83,21 +85,22 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable applicationRepository.configserverConfig().hostedVespa() ? VipStatusMode.VIP_STATUS_FILE : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY, - flagSource, convergence); + flagSource, convergence, Clock.systemUTC()); } // For testing only ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, VipStatusMode vipStatusMode, - FlagSource flagSource, ConfigConvergenceChecker convergence) { + FlagSource flagSource, ConfigConvergenceChecker convergence, Clock clock) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, - FOR_TESTING_NO_BOOTSTRAP_OF_APPS, CONTINUE, vipStatusMode, flagSource, convergence); + FOR_TESTING_NO_BOOTSTRAP_OF_APPS, CONTINUE, vipStatusMode, flagSource, convergence, clock); } private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails, - VipStatusMode vipStatusMode, FlagSource flagSource, ConfigConvergenceChecker convergence) { + VipStatusMode vipStatusMode, FlagSource flagSource, ConfigConvergenceChecker convergence, + Clock clock) { this.applicationRepository = applicationRepository; this.server = server; this.versionState = versionState; @@ -107,6 +110,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap()); this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; + this.clock = clock; rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); configServerMaintenance = new ConfigServerMaintenance(configserverConfig, @@ -197,8 +201,8 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private void startRpcServerWithFileDistribution() { rpcServerExecutor.execute(server); - Instant end = Instant.now().plus(Duration.ofSeconds(10)); - while (!server.isRunning() && Instant.now().isBefore(end)) { + Instant end = clock.instant().plus(Duration.ofSeconds(10)); + while (!server.isRunning() && clock.instant().isBefore(end)) { try { Thread.sleep(10); } catch (InterruptedException e) { @@ -220,7 +224,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } private void redeployAllApplications() throws InterruptedException { - Instant end = Instant.now().plus(maxDurationOfRedeployment); + Instant end = clock.instant().plus(maxDurationOfRedeployment); List<ApplicationId> applicationsToRedeploy = applicationRepository.listApplications(); Collections.shuffle(applicationsToRedeploy); long failCount = 0; @@ -233,7 +237,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable log.log(Level.INFO, "Redeployment of " + applicationsToRedeploy + " not finished, will retry in " + sleepTime); Thread.sleep(sleepTime.toMillis()); } - } while ( ! applicationsToRedeploy.isEmpty() && Instant.now().isBefore(end)); + } while ( ! applicationsToRedeploy.isEmpty() && clock.instant().isBefore(end)); if ( ! applicationsToRedeploy.isEmpty()) throw new RuntimeException("Redeploying applications not finished after " + maxDurationOfRedeployment + @@ -296,11 +300,11 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } private void logProgress(LogState logState, int failedDeployments, int finishedDeployments) { - if ( ! Duration.between(logState.lastLogged, Instant.now()).minus(Duration.ofSeconds(10)).isNegative() + if ( ! Duration.between(logState.lastLogged, clock.instant()).minus(Duration.ofSeconds(10)).isNegative() && (logState.failedDeployments != failedDeployments || logState.finishedDeployments != finishedDeployments)) { log.log(Level.INFO, () -> finishedDeployments + " of " + logState.applicationCount + " apps redeployed " + "(" + failedDeployments + " failed)"); - logState.update(Instant.now(), failedDeployments, finishedDeployments); + logState.update(clock.instant(), failedDeployments, finishedDeployments); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 98ddf702a7f..937ec2bb0e7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -472,7 +472,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica // If some are missing, quorum is enough, but wait for all up to 5 seconds before returning if (respondents.size() >= barrierMemberCount()) { if (gotQuorumTime.isBefore(startTime)) - gotQuorumTime = Instant.now(); + gotQuorumTime = clock.instant(); // Give up if more than some time has passed since we got quorum, otherwise continue if (Duration.between(Instant.now(), gotQuorumTime.plus(waitForAll)).isNegative()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index b3c2fa2e300..7d53c383cf5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -330,7 +330,7 @@ public class TenantRepository { private Tenant createTenant(TenantName tenantName, Instant created) { if (tenants.containsKey(tenantName)) return getTenant(tenantName); - Instant start = Instant.now(); + Instant start = clock.instant(); log.log(Level.FINE, () -> "Adding tenant '" + tenantName); TenantApplications applicationRepo = new TenantApplications(tenantName, @@ -375,7 +375,7 @@ public class TenantRepository { configDefinitionRepo, zookeeperServerConfig.juteMaxBuffer()); log.log(Level.INFO, "Adding tenant '" + tenantName + "'" + ", created " + created + - ". Bootstrapping in " + Duration.between(start, Instant.now())); + ". Bootstrapping in " + Duration.between(start, clock.instant())); Tenant tenant = new Tenant(tenantName, sessionRepository, applicationRepo, created); createAndWriteTenantMetaData(tenant); tenants.putIfAbsent(tenantName, tenant); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index babc7b79b65..67613d5a806 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -18,6 +18,7 @@ import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.path.Path; +import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.deploy.DeployTester; @@ -42,6 +43,7 @@ import java.util.Optional; import java.util.function.BooleanSupplier; import java.util.stream.Collectors; +import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode; import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_FILE; import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_PROGRAMMATICALLY; import static com.yahoo.vespa.config.server.deploy.DeployTester.createHostedModelFactory; @@ -56,6 +58,7 @@ import static org.junit.Assert.assertTrue; public class ConfigServerBootstrapTest { private final MockCurator curator = new MockCurator(); + private final ManualClock clock = new ManualClock(); @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -177,7 +180,7 @@ public class ConfigServerBootstrapTest { waitUntil(() -> bootstrap.vipStatus().isInRotation(), "failed waiting for server to be in rotation"); } - private ConfigServerBootstrap createBootstrap(DeployTester tester, RpcServer rpcServer, ConfigServerBootstrap.VipStatusMode vipStatusProgrammatically) throws IOException { + private ConfigServerBootstrap createBootstrap(DeployTester tester, RpcServer rpcServer, VipStatusMode vipStatusMode) throws IOException { VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); @@ -188,9 +191,10 @@ public class ConfigServerBootstrapTest { versionState, stateMonitor, vipStatus, - vipStatusProgrammatically, + vipStatusMode, new InMemoryFlagSource(), - new ConfigConvergenceChecker()); + new ConfigConvergenceChecker(), + clock); } private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException { @@ -223,7 +227,7 @@ public class ConfigServerBootstrapTest { .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .hostedVespa(hosted) .multitenant(hosted) - .maxDurationOfBootstrap(2) /* seconds */ + .maxDurationOfBootstrap(0) /* seconds, 0 => it will not retry deployment if bootstrap fails */ .sleepTimeWhenRedeployingFails(0)); /* seconds */ } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 776a0a29b7d..f2117764be1 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -503,7 +503,8 @@ "public void <init>(com.yahoo.prelude.query.Item, java.util.Collection)", "public com.yahoo.prelude.query.Item$ItemType getItemType()", "public java.lang.String getName()", - "protected void adding(com.yahoo.prelude.query.Item)" + "protected void adding(com.yahoo.prelude.query.Item)", + "public static boolean acceptsChildrenOfType(com.yahoo.prelude.query.Item$ItemType)" ], "fields": [] }, @@ -1594,6 +1595,7 @@ "public boolean hasItemClass(java.lang.Class)", "public com.yahoo.prelude.query.Item createItemClass()", "public java.lang.String toSign()", + "public com.yahoo.prelude.query.Item$ItemType toItemType()", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 51c82eac264..ae953426e63 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -348,7 +348,7 @@ public class ClusterSearcher extends Searcher { if (query.getOffset() > 0 || query.getHits() < mergedResult.hits().size()) { if (mergedResult.getHitOrderer() != null) { // Make sure we have the necessary data for sorting - searcher.fill(mergedResult, Execution.ATTRIBUTEPREFETCH, execution); + searcher.fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS, execution); } mergedResult.hits().trim(query.getOffset(), query.getHits()); query.setOffset(0); // Needed when doing a trim diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 1a92e388f47..7abaf99c93d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -37,6 +37,9 @@ import java.util.logging.Logger; */ public abstract class VespaBackEndSearcher extends PingableSearcher { + /** for vespa-internal use only; consider renaming the summary class */ + public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; + private static final CompoundName TRACE_DISABLE = new CompoundName("trace.disable"); private String serverId; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java index 55adc3cba8d..0c4dff15744 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java @@ -22,9 +22,7 @@ public abstract class CompositeIndexedItem extends CompositeTaggableItem impleme private String index = ""; - /** - * The name of the index this belongs to, or "" (never null) if not specified - **/ + /** The name of the index this belongs to, or "" (never null) if not specified */ public String getIndexName() { return index; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java b/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java index 97cbfac0b2d..500c68cf4bd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java @@ -83,11 +83,16 @@ public class EquivItem extends CompositeTaggableItem { super.adding(item); Validator.ensure("Could not add an item of type " + item.getItemType() + ": Equiv can only have word, wordAlternatives, int, exact, or phrase as children", - item.getItemType() == ItemType.WORD || - item.getItemType() == ItemType.WORD_ALTERNATIVES || - item.getItemType() == ItemType.INT || - item.getItemType() == ItemType.EXACT || - item.getItemType() == ItemType.PHRASE); + acceptsChildrenOfType(item.getItemType())); + } + + /** Returns true if this accepts child items of the given type */ + public static boolean acceptsChildrenOfType(ItemType itemType) { + return itemType == ItemType.WORD || + itemType == ItemType.WORD_ALTERNATIVES || + itemType == ItemType.INT || + itemType == ItemType.EXACT || + itemType == ItemType.PHRASE; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java index 309befd80f5..fbbc746b130 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java @@ -3,35 +3,41 @@ package com.yahoo.prelude.query; /** - * A term type enumeration + * A term type enumeration. * * @author bratseth * @author Steinar Knutsen */ public class TermType { - public static final TermType RANK = new TermType("rank", RankItem.class, null, "$"); + public static final TermType RANK = new TermType("rank", Item.ItemType.RANK, RankItem.class, null, "$"); - public static final TermType AND = new TermType("and", AndItem.class, null, "+"); + public static final TermType AND = new TermType("and", Item.ItemType.AND, AndItem.class, null, "+"); - public static final TermType OR = new TermType("or", OrItem.class, null, "?"); + public static final TermType OR = new TermType("or", Item.ItemType.OR, OrItem.class, null, "?"); - public static final TermType NOT = new TermType("not", NotItem.class, null, "-"); + public static final TermType NOT = new TermType("not", Item.ItemType.NOT, NotItem.class, null, "-"); - public static final TermType PHRASE = new TermType("phrase", PhraseItem.class, null, "\""); + public static final TermType PHRASE = new TermType("phrase", Item.ItemType.PHRASE, PhraseItem.class, null, "\""); - public static final TermType EQUIV = new TermType("equiv", EquivItem.class, null, ""); + public static final TermType EQUIV = new TermType("equiv", Item.ItemType.EQUIV, EquivItem.class, null, ""); - public static final TermType DEFAULT = new TermType("", CompositeItem.class, AndItem.class, ""); + public static final TermType DEFAULT = new TermType("", Item.ItemType.AND, CompositeItem.class, AndItem.class, ""); public final String name; + private final Item.ItemType itemType; private final String sign; private final Class<? extends CompositeItem> instanceClass; private final Class<? extends CompositeItem> itemClass; - private TermType(String name, Class<? extends CompositeItem> itemClass, Class<? extends CompositeItem> instanceClass, String sign) { + private TermType(String name, + Item.ItemType itemType, + Class<? extends CompositeItem> itemClass, + Class<? extends CompositeItem> instanceClass, + String sign) { this.name = name; + this.itemType = itemType; this.itemClass = itemClass; if (instanceClass == null) { this.instanceClass = itemClass; @@ -73,6 +79,8 @@ public class TermType { return sign; } + public Item.ItemType toItemType() { return itemType; } + @Override public boolean equals(Object o) { if ( ! (o instanceof TermType)) return false; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 932f4ea7c38..5dcd591b1fc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -8,6 +8,7 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -113,7 +114,8 @@ public class BlendingSearcher extends Searcher { private Result sortAndTrimResults(Result result, Query q, int offset, int hits, Execution execution) { if (q.getRanking().getSorting() != null) { - execution.fillAttributes(result); // Always correct as we can only sort on attributes + // TODO: remove or rename this internal summary class for Vespa 9 + execution.fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); result.hits().sort(); } result.hits().trim(offset, hits); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index 8e137d99951..4980b035876 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -1,10 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics; -import com.yahoo.language.Language; import com.yahoo.language.Linguistics; -import com.yahoo.language.process.StemMode; -import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.semantics.rule.CompositeCondition; import com.yahoo.prelude.semantics.rule.Condition; import com.yahoo.prelude.semantics.rule.NamedCondition; @@ -76,12 +73,9 @@ public class RuleBase { */ private boolean usesAutomata = false; - private RuleBaseLinguistics linguistics; - /** Creates an empty rule base */ - public RuleBase(String name, Linguistics linguistics) { + public RuleBase(String name) { this.name = name; - this.linguistics = new RuleBaseLinguistics(StemMode.BEST, Language.ENGLISH, linguistics); } /** @@ -284,7 +278,7 @@ public class RuleBase { } /** - * Set to truew if this uses an automata, even if an automata is not present right now. + * Set to true if this uses an automata, even if an automata is not present right now. * Useful to validate without having automatas available */ void setUsesAutomata(boolean usesAutomata) { this.usesAutomata = usesAutomata; } @@ -342,7 +336,7 @@ public class RuleBase { } // TODO: Values are not added right now - protected void annotatePhrase(PhraseMatcher.Phrase phrase,Query query,int traceLevel) { + protected void annotatePhrase(PhraseMatcher.Phrase phrase, Query query, int traceLevel) { for (StringTokenizer tokens = new StringTokenizer(phrase.getData(), "|", false); tokens.hasMoreTokens(); ) { String token = tokens.nextToken(); int semicolonIndex = token.indexOf(";"); @@ -357,12 +351,12 @@ public class RuleBase { phrase.getItem(0).addAnnotation(annotation, phrase); if (traceLevel >= 4) query.trace(" Annotating '" + phrase + "' as " + annotation + - (value.equals("") ? "" :"=" + value),false,1); + (value.equals("") ? "" :"=" + value),false,1); } } private void makeReferences() { - for (Iterator<ProductionRule> i=ruleIterator(); i.hasNext(); ) { + for (Iterator<ProductionRule> i = ruleIterator(); i.hasNext(); ) { ProductionRule rule = i.next(); rule.makeReferences(this); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java index acbf9a7ffb6..6e7286cb8dc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java @@ -99,7 +99,7 @@ public class RuleImporter { reader = IOUtils.createReader(fileName, "utf-8"); File file = new File(fileName); String absoluteFileName = file.getAbsolutePath(); - var ruleBase = new RuleBase(stripLastName(file.getName()), linguistics); + var ruleBase = new RuleBase(stripLastName(file.getName())); privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase); return ruleBase; } @@ -205,7 +205,7 @@ public class RuleImporter { /** Imports an unitialized rule base */ public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException { if (config == null) throw new IllegalStateException("Must initialize with config if importing from config"); - RuleBase ruleBase = new RuleBase(ruleBaseConfig.name(), linguistics); + RuleBase ruleBase = new RuleBase(ruleBaseConfig.name()); return privateImportFromReader(new StringReader(ruleBaseConfig.rules()), "semantic-rules.cfg", ruleBaseConfig.automata(),ruleBase); @@ -234,7 +234,7 @@ public class RuleImporter { public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { try { if (ruleBase == null) - ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName, linguistics); + ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName); ruleBase.setSource(sourceName.replace('\\', '/')); new SemanticsParser(reader, linguistics).semanticRules(ruleBase, this); if (automataFile != null && !automataFile.isEmpty()) diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java index 85bcdf8bd20..d5ed89b0724 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.semantics.engine; import com.yahoo.prelude.query.*; +import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.search.Query; import com.yahoo.search.query.QueryTree; @@ -32,11 +33,13 @@ public class Evaluation { /** The rule evaluation context, can be reset once the rule is evaluated */ private final RuleEvaluation ruleEvaluation; + private final RuleBase ruleBase; + /** * The amount of context information to collect about this evaluation. * 0 means no context information, higher numbers means more context information. */ - private int traceLevel = 0; + private final int traceLevel; private String traceIndentation = ""; @@ -49,8 +52,8 @@ public class Evaluation { /** Should we allow stemmed matches? */ private boolean stemming = true; - public Evaluation(Query query) { - this(query,0); + public Evaluation(Query query, RuleBase ruleBase) { + this(query, ruleBase, 0); } /** @@ -59,13 +62,17 @@ public class Evaluation { * @param query the query this evaluation is for * @param traceLevel the amount of tracing to do */ - public Evaluation(Query query, int traceLevel) { + public Evaluation(Query query, RuleBase ruleBase, int traceLevel) { this.query = query; + this.ruleBase = ruleBase; this.traceLevel = traceLevel; reset(); ruleEvaluation = new RuleEvaluation(this); } + /** Returns the rule base this evaluates over */ + public RuleBase ruleBase() { return ruleBase; } + /** Resets the item iterator to point to the first item */ public void reset() { if (flattenedItems != null) @@ -134,11 +141,11 @@ public class Evaluation { /** Adds an item to the query being evaluated in a way consistent with the query type */ // TODO: Add this functionality to Query? public void addItem(Item item, TermType termType) { - Item root= query.getModel().getQueryTree().getRoot(); - if (root==null) + Item root = query.getModel().getQueryTree().getRoot(); + if (root == null) query.getModel().getQueryTree().setRoot(item); else - query.getModel().getQueryTree().setRoot(combineItems(root,item,termType)); + query.getModel().getQueryTree().setRoot(combineItems(root, item, termType)); } /** Removes this item */ @@ -151,18 +158,18 @@ public class Evaluation { * equal items */ public void removeItemByIdentity(Item item) { - int position=findIndexByIdentity(item); - if (position>=0) + int position = findIndexByIdentity(item); + if (position >= 0) item.getParent().removeItem(position); else item.getParent().removeItem(item); // Fallback to removeField by equal() } private int findIndexByIdentity(Item item) { - int position=0; - for (Iterator<Item> i=item.getParent().getItemIterator(); i.hasNext(); ) { - Item child=i.next(); - if (item==child) { + int position = 0; + for (Iterator<Item> i = item.getParent().getItemIterator(); i.hasNext(); ) { + Item child = i.next(); + if (item == child) { return position; } position++; @@ -172,7 +179,7 @@ public class Evaluation { /** Removes an item, prefers the one at/close to the given position if there are multiple ones */ public void removeItem(int position,Item item) { - Item removeCandidate=item.getParent().getItem(position); + Item removeCandidate = item.getParent().getItem(position); if (removeCandidate.equals(item)) // Remove based on position item.getParent().removeItem(position); else @@ -189,7 +196,7 @@ public class Evaluation { if (!(item instanceof SegmentItem)) { return item; } - CompositeItem converted = null; + CompositeItem converted; if (item instanceof AndSegmentItem) { converted = new AndItem(); } else if (item instanceof PhraseSegmentItem) { @@ -238,16 +245,21 @@ public class Evaluation { /** * Inserts an item to the query being evaluated in a way consistent with the query type * - * @param item the item to insert - * @param parent the parent of this item, or null to set the root - * @param index the index at which to insert this into the parent - * @param desiredParentType the desired type of the composite which contains item when this returns + * @param items the items to insert + * @param parent the parent of these items, or null to set the root + * @param index the index at which to insert these into the parent + * @param desiredParentType the desired type of the composite which contains items when this returns */ - public void insertItem(Item item, CompositeItem parent, int index, TermType desiredParentType) { + public void insertItems(List<Item> items, CompositeItem parent, int index, TermType desiredParentType) { if (isEmpty(parent)) { - CompositeItem newParent = (CompositeItem)desiredParentType.createItemClass(); - newParent.addItem(item); - query.getModel().getQueryTree().setRoot(newParent); + if (items.size() == 1 && desiredParentType.hasItemClass(items.get(0).getClass())) { + query.getModel().getQueryTree().setRoot(items.get(0)); + } + else { + CompositeItem newParent = (CompositeItem) desiredParentType.createItemClass(); + items.forEach(item -> newParent.addItem(item)); + query.getModel().getQueryTree().setRoot(newParent); + } return; } @@ -260,34 +272,37 @@ public class Evaluation { } if (( desiredParentType == TermType.DEFAULT || desiredParentType.hasItemClass(parent.getClass()) ) - && equalIndexNameIfParentIsPhrase(item, parent)) { - addItem(parent, index, item, desiredParentType); + && equalIndexNameIfParentIsPhrase(items, parent)) { + for (Item item : items) + addItem(parent, index, item, desiredParentType); } - else if (incompatible(desiredParentType, parent)) { - insertIncompatibleItem(item, parent, query, desiredParentType); + else if (parent.items().isEmpty()) { + CompositeItem parentsParent = parent.getParent(); + CompositeItem newParent = newParent(desiredParentType); + items.forEach(item -> newParent.addItem(item)); + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); + } + else if (items.size() == 1 && desiredParentType.hasItemClass(items.get(0).getClass())) { + addItem(parent, index, items.get(0), desiredParentType); } else { - insertIncompatibleItemAsParent(item, parent, query, desiredParentType); + insertWithDesiredParentType(items, parent, desiredParentType); } } + /** Returns true if this item represents an empty query *tree*. */ private boolean isEmpty(Item item) { if (item == null) return true; if (item instanceof QueryTree && ((QueryTree) item).isEmpty()) return true; return false; } - /** Returns true if the desired type cannot have childCandidate as a child */ - private boolean incompatible(TermType desiredParentType, Item childCandidate) { - return desiredParentType == TermType.EQUIV && childCandidate.getItemType() != Item.ItemType.EQUIV; - } - private void addItem(CompositeItem parent, int index, Item item, TermType desiredParentType) { if (parent instanceof NotItem) { if (index == 0 && parent.getItem(0) == null) { // Case 1: The current positive is null and we are adding a positive parent.setItem(0, item); } - else if (index<=1 && !(parent.getItem(0) instanceof CompositeItem)) { // Case 2: The positive must become a composite + else if (index <= 1 && !(parent.getItem(0) instanceof CompositeItem)) { // Case 2: The positive must become a composite CompositeItem positiveComposite = (CompositeItem)desiredParentType.createItemClass(); positiveComposite.addItem(parent.getItem(0)); positiveComposite.addItem(index, item); @@ -313,50 +328,45 @@ public class Evaluation { } /** A special purpose check used to simplify the above */ - private boolean equalIndexNameIfParentIsPhrase(Item item,CompositeItem parent) { + private boolean equalIndexNameIfParentIsPhrase(List<Item> items, CompositeItem parent) { if ( ! (parent instanceof PhraseItem)) return true; - if ( ! (item instanceof IndexedItem)) return true; - - return ((PhraseItem)parent).getIndexName().equals(((IndexedItem)item).getIndexName()); - } - - private void insertIncompatibleItem(Item item, CompositeItem parent, Query query, TermType desiredParentType) { - CompositeItem newParent; - if (desiredParentType == TermType.DEFAULT) - newParent = new AndItem(); - else - newParent = (CompositeItem)desiredParentType.createItemClass(); + var phrase = (PhraseItem)parent; - newParent.addItem(item); - parent.addItem(newParent); + for (Item item : items) { + if ( ! (item instanceof IndexedItem)) continue; + var indexedItem = (IndexedItem)item; + if (! indexedItem.getIndexName().equals(phrase.getIndexName())) return false; + } + return true; } - private void insertIncompatibleItemAsParent(Item item, CompositeItem parent, Query query, TermType desiredParentType) { - // Create new parent - CompositeItem newParent; - if (desiredParentType == TermType.DEFAULT) - newParent = new AndItem(); - else - newParent = (CompositeItem)desiredParentType.createItemClass(); - - // Save previous parent parent + private void insertWithDesiredParentType(List<Item> items, CompositeItem parent, TermType desiredParentType) { CompositeItem parentsParent = parent.getParent(); - // Add items to new parent - newParent.addItem(parent); - newParent.addItem(item); + CompositeItem newParent = newParent(desiredParentType); + + if (! (parentsParent instanceof QueryTree) && parentsParent.getItemType() == newParent.getItemType()) { // Collapse + newParent = parentsParent; + } - // Insert new parent as root or child of old parents parent - if (parentsParent == null) { - query.getModel().getQueryTree().setRoot(newParent); + for (Item item : items) + newParent.addItem(item); + if (desiredParentType == TermType.EQUIV || desiredParentType == TermType.PHRASE) { // insert new parent below the current + parent.addItem(newParent); } - else { - parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); + else { // insert new parent above the current + newParent.addItem(parent); + if (newParent != parentsParent) // Insert new parent as root or child of old parent's parent + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); } } - private Item combineItems(Item first,Item second,TermType termType) { + private CompositeItem newParent(TermType desiredParentType) { + return desiredParentType == TermType.DEFAULT ? new AndItem() : (CompositeItem)desiredParentType.createItemClass(); + } + + private Item combineItems(Item first, Item second, TermType termType) { if (first instanceof NullItem) { return second; } else if (first instanceof NotItem) { @@ -378,6 +388,10 @@ public class Evaluation { return composite; } else { + if (combined instanceof EquivItem) { + first = makeEquivCompatible(first); + second = makeEquivCompatible(second); + } combined.addItem(first); combined.addItem(second); // Also works for nots return combined; @@ -394,6 +408,22 @@ public class Evaluation { } } + private Item makeEquivCompatible(Item item) { + if (item instanceof AndItem || item instanceof WeakAndItem) { + PhraseItem phrase = new PhraseItem(); + List<Item> children = ((CompositeItem)item).items(); + if (children.isEmpty()) return phrase; + String index = ((IndexedItem)children.get(0)).getIndexName(); + for (var child : ((CompositeItem)item).items()) + phrase.addItem(child); + phrase.setIndexName(index); + return phrase; + } + else { + return item; // Compatible, or can't be made so + } + } + private CompositeItem createType(TermType termType) { if (termType == TermType.DEFAULT) { if (query.getModel().getType() == Query.Type.ANY) diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java index 6098fbee327..6ed09926ccb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java @@ -60,7 +60,12 @@ public class Match { /** Returns a new item representing this match */ public Item toItem(String label) { - var newItem = new WordItem(getReplaceValue(), label); + return toItem(label, getReplaceValue()); + } + + /** Returns a new item representing this match */ + public Item toItem(String label, String term) { + var newItem = new WordItem(term, label); newItem.setWeight(item.getWeight()); return newItem; } @@ -81,4 +86,9 @@ public class Match { return true; } + @Override + public String toString() { + return "match of " + item + " at " + position + " to be replaced by " + replaceValue; + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java index 605f3a23e10..b7083414e85 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java @@ -8,7 +8,7 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PhraseItem; /** - * The Matches referenced by a particular context name in a rule evaluation + * The matches referenced by a particular context name in a rule evaluation * * @author bratseth */ diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java index dd6610d1184..e2756afbb2e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java @@ -37,7 +37,7 @@ public class RuleEngine { // Probably create indices on the first term like Prolog implementations use to boolean matchedAnything = false; - Evaluation evaluation = new Evaluation(query, traceLevel); + Evaluation evaluation = new Evaluation(query, rules, traceLevel); if (traceLevel >= 2) evaluation.trace(2,"Evaluating query '" + evaluation.getQuery().getModel().getQueryTree().getRoot() + "':"); for (ListIterator<ProductionRule> i = rules.ruleIterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java index 2a00843a85c..1da16b4d277 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java @@ -190,8 +190,9 @@ public class RuleEvaluation { // TODO: Simplistic yet. Nedd to support context nesting etc. public void entering(String context) { if (context == null) return; - if (matchReferences != null && matchReferences.contains(context)) + if (matchReferences != null && matchReferences.contains(context)) { currentContext = context; + } } public void leaving(String context) { @@ -209,7 +210,7 @@ public class RuleEvaluation { */ public void addMatch(FlattenedItem item, String replaceString) { evaluation.makeParentMutable(item.getItem()); - Match match = new Match(item,replaceString); + Match match = new Match(item, replaceString); if (currentContext != null) { ReferencedMatches matches = getReferencedMatches(currentContext); if (matches == null) { @@ -241,8 +242,8 @@ public class RuleEvaluation { public Evaluation getEvaluation() { return evaluation; } /** Adds an item to the query being evaluated in a way consistent with the query type */ - public void addItem(Item item, TermType termType) { - evaluation.addItem(item,termType); + public void addItems(List<Item> items, TermType termType) { + items.forEach(item -> evaluation.addItem(item, termType)); } public void removeItem(Item item) { @@ -262,13 +263,13 @@ public class RuleEvaluation { /** * Inserts an item to the query being evaluated in a way consistent with the query type * - * @param item the item to insert + * @param items the items to insert * @param parent the parent of this item, or null to set the root * @param index the index at which to insert this into the parent * @param termType the kind of item to index, this decides the resulting structure */ - public void insertItem(Item item, CompositeItem parent, int index, TermType termType) { - evaluation.insertItem(item, parent, index, termType); + public void insertItems(List<Item> items, CompositeItem parent, int index, TermType termType) { + evaluation.insertItems(items, parent, index, termType); } /** Returns a read-only view of the items of this */ diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java index 77bffa5778e..a49272a73ea 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -14,28 +15,28 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; */ public abstract class CompositeCondition extends Condition { - private List<Condition> conditions=new java.util.ArrayList<>(); + private final List<Condition> conditions = new java.util.ArrayList<>(); public CompositeCondition() { } public void preMatchHook(RuleEvaluation e) { super.preMatchHook(e); - if (e.getTraceLevel()>=3) { - e.trace(3,"Evaluating '" + this + "'" + " at " + e.currentItem()); + if (e.getTraceLevel() >= 3) { + e.trace(3, "Evaluating '" + this + "'" + " at " + e.currentItem()); e.indentTrace(); } } public void postMatchHook(RuleEvaluation e) { - if (e.getTraceLevel()>=3) { + if (e.getTraceLevel() >= 3) { e.unindentTrace(); } } protected boolean hasOpenChoicepoint(RuleEvaluation evaluation) { - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { - Condition subCondition=i.next(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { + Condition subCondition = i.next(); if (subCondition.hasOpenChoicepoint(evaluation)) return true; } @@ -48,8 +49,8 @@ public abstract class CompositeCondition extends Condition { } /** Sets the condition at the given index */ - public void setCondition(int index,Condition condition) { - conditions.set(index,condition); + public void setCondition(int index, Condition condition) { + conditions.set(index, condition); } /** Returns the number of subconditions */ @@ -74,7 +75,7 @@ public abstract class CompositeCondition extends Condition { * @throws IndexOutOfBoundsException if there is no condition at this index */ public Condition removeCondition(int i) { - Condition condition=conditions.remove(i); + Condition condition = conditions.remove(i); condition.setParent(null); return condition; } @@ -82,20 +83,22 @@ public abstract class CompositeCondition extends Condition { /** Returns an iterator of the immediate children of this condition */ public Iterator<Condition> conditionIterator() { return conditions.iterator(); } + public List<Condition> conditions() { return Collections.unmodifiableList(conditions); } + public void makeReferences(RuleBase rules) { - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { - Condition condition=i.next(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { + Condition condition = i.next(); condition.makeReferences(rules); } } /** Whether this should be output with parentheses, default is parent!=null */ protected boolean useParentheses() { - return getParent()!=null; + return getParent() != null; } protected String toInnerString(String conditionSeparator) { - if (getLabel()!=null) + if (getLabel() != null) return getLabel() + ":(" + conditionsToString(conditionSeparator) + ")"; else if (useParentheses()) return "(" + conditionsToString(conditionSeparator) + ")"; @@ -104,8 +107,8 @@ public abstract class CompositeCondition extends Condition { } protected final String conditionsToString(String conditionSeparator) { - StringBuilder buffer=new StringBuilder(); - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { + StringBuilder buffer = new StringBuilder(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { buffer.append(i.next().toString()); if (i.hasNext()) buffer.append(conditionSeparator); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java index 0d3c93619b5..8d6016cb060 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java @@ -14,20 +14,20 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; public abstract class Condition { /** The parent of this condition, or null if this is not nested */ - private CompositeCondition parent=null; + private CompositeCondition parent = null; /** * The label of this condition, or null if none. * Specified by label:condition * The label is also the default context is no context is speficied explicitly */ - private String label=null; + private String label; /** * The name space refered by this match, or null if the default (query) * Specified by namespace.condition in rules. */ - private String nameSpace=null; + private String nameSpace = null; /** * The name of the context created by this, or null if none @@ -36,9 +36,9 @@ public abstract class Condition { private String contextName; /** Position constraints of the terms matched by this condition */ - private Anchor anchor=Anchor.NONE; + private Anchor anchor = Anchor.NONE; - public static enum Anchor { + public enum Anchor { NONE, START, END, BOTH; public static Anchor create(boolean start,boolean end) { if (start && end) return Anchor.BOTH; @@ -49,32 +49,32 @@ public abstract class Condition { } public Condition() { - this(null,null); + this(null, null); } public Condition(String label) { - this(label,null); + this(label, null); } - public Condition(String label,String context) { - this.label=label; - this.contextName=context; + public Condition(String label, String context) { + this.label = label; + this.contextName = context; } /** * Sets the name whatever is matched by this condition can be refered as, or null - * to make it unreferable + * to make it nonreferable */ - public void setContextName(String contextName) { this.contextName=contextName; } + public void setContextName(String contextName) { this.contextName = contextName; } /** - * Returns the name whatever is matched by this condition can be refered as, or null + * Returns the name whatever is matched by this condition can be referred as, or null * if it is unreferable */ public String getContextName() { return contextName; } /** Returns whether this is referable, returns context!=null by default */ - protected boolean isReferable() { return contextName!=null; } + protected boolean isReferable() { return contextName != null; } /** Sets the label of this. Set to null to use the default */ public String getLabel() { return label; } @@ -95,15 +95,15 @@ public abstract class Condition { void setParent(CompositeCondition parent) { this.parent=parent; } /** Sets a positional constraint on this condition */ - public void setAnchor(Anchor anchor) { this.anchor=anchor; } + public void setAnchor(Anchor anchor) { this.anchor = anchor; } /** Returns the positional constraint on this anchor. This is never null */ public Anchor getAnchor() { return anchor; } /** - * <p>Returns whether this condition matches the given evaluation + * Returns whether this condition matches the given evaluation * at the <i>current</i> location of the evaluation. Calls the doesMatch - * method of each condition subtype.</p> + * method of each condition subtype. */ public final boolean matches(RuleEvaluation e) { // TODO: With this algoritm, each choice point will move to the next choice on each reevaluation @@ -138,32 +138,32 @@ public abstract class Condition { /** Check start anchor. Trace level 4 if no match */ protected boolean matchesStartAnchor(RuleEvaluation e) { - if (anchor!=Anchor.START && anchor!=Anchor.BOTH) return true; - if (e.getPosition()==0) return true; - if (e.getTraceLevel()>=4) - e.trace(4,this + " must be at the start, which " + e.currentItem() + " isn't"); + if (anchor != Anchor.START && anchor != Anchor.BOTH) return true; + if (e.getPosition() == 0) return true; + if (e.getTraceLevel() >= 4) + e.trace(4, this + " must be at the start, which " + e.currentItem() + " isn't"); return false; } /** Check start anchor. Trace level 4 if no match */ protected boolean matchesEndAnchor(RuleEvaluation e) { - if (anchor!=Anchor.END && anchor!=Anchor.BOTH) return true; - if (e.getPosition()>=e.items().size()) return true; - if (e.getTraceLevel()>=4) - e.trace(4,this + " must be at the end, which " + e.currentItem() + " isn't"); + if (anchor != Anchor.END && anchor != Anchor.BOTH) return true; + if (e.getPosition() >= e.items().size()) return true; + if (e.getTraceLevel() >= 4) + e.trace(4, this + " must be at the end, which " + e.currentItem() + " isn't"); return false; } - protected void traceResult(boolean matches,RuleEvaluation e) { - if (matches && e.getTraceLevel()>=3) - e.trace(3,"Matched '" + this + "'" + getMatchInfoString(e) + " at " + e.previousItem()); - if (!matches && e.getTraceLevel()>=4) - e.trace(4,"Did not match '" + this + "' at " + e.currentItem()); + protected void traceResult(boolean matches, RuleEvaluation e) { + if (matches && e.getTraceLevel() >= 3) + e.trace(3, "Matched '" + this + "'" + getMatchInfoString(e) + " at " + e.previousItem()); + if (!matches && e.getTraceLevel() >= 4) + e.trace(4, "Did not match '" + this + "' at " + e.currentItem()); } protected String getMatchInfoString(RuleEvaluation e) { - String matchInfo=getMatchInfo(e); - if (matchInfo==null) return ""; + String matchInfo = getMatchInfo(e); + if (matchInfo == null) return ""; return " as '" + matchInfo + "'"; } @@ -212,28 +212,28 @@ public abstract class Condition { public void makeReferences(RuleBase rules) { } protected String getLabelString() { - if (label==null) return ""; + if (label == null) return ""; return label + ":"; } /** Whether the label matches the current item, true if there is no current item */ protected boolean labelMatches(RuleEvaluation e) { - FlattenedItem flattenedItem=e.currentItem(); - if (flattenedItem==null) return true; - TermItem item=flattenedItem.getItem(); - if (item==null) return true; - return labelMatches(item,e); - } - - protected boolean labelMatches(TermItem evaluationTerm,RuleEvaluation e) { - String indexName=evaluationTerm.getIndexName(); - String label=getLabel(); - if (label==null) - label=e.getCurrentLabel(); - if ("".equals(indexName) && label==null) return true; + FlattenedItem flattenedItem = e.currentItem(); + if (flattenedItem == null) return true; + TermItem item = flattenedItem.getItem(); + if (item == null) return true; + return labelMatches(item, e); + } + + protected boolean labelMatches(TermItem evaluationTerm, RuleEvaluation e) { + String indexName = evaluationTerm.getIndexName(); + String label = getLabel(); + if (label == null) + label = e.getCurrentLabel(); + if ("".equals(indexName) && label == null) return true; if (indexName.equals(label)) return true; - if (e.getTraceLevel()>=4) - e.trace(4,"'" + this + "' does not match, label of " + e.currentItem() + " was required to be " + label); + if (e.getTraceLevel() >= 4) + e.trace(4, "'" + this + "' does not match, label of " + e.currentItem() + " was required to be " + label); return false; } @@ -242,13 +242,14 @@ public abstract class Condition { protected boolean isDefaultContextName() { return false; } + @Override public String toString() { - String contextString=""; - String nameSpaceString=""; - if (contextName!=null && !isDefaultContextName()) - contextString=contextName + "/"; - if (getNameSpace()!=null) - nameSpaceString=getNameSpace() + "."; + String contextString = ""; + String nameSpaceString = ""; + if (contextName != null && !isDefaultContextName()) + contextString = contextName + "/"; + if (getNameSpace() != null) + nameSpaceString = getNameSpace() + "."; return contextString + nameSpaceString + toInnerString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java index bfddb55566c..e468526fbe0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; -import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.querytransform.PhraseMatcher; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; @@ -11,8 +10,6 @@ import com.yahoo.prelude.semantics.engine.FlattenedItem; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; -import java.util.Map; - /** * A reference to a named condition * @@ -24,7 +21,7 @@ public class ConditionReference extends Condition { private String conditionName; /** - * The actual condition references by this, or null if not initialized or not found, + * The actual condition referenced by this, or null if not initialized or not found, * or if this is really an automata reference */ private NamedCondition namedCondition; @@ -33,7 +30,7 @@ public class ConditionReference extends Condition { * True if this condition should be looked up in the automata * annotations of the item instead of by reference to another item */ - private boolean automataLookup=false; + private boolean automataLookup = false; public ConditionReference(String conditionName) { this(null,conditionName); @@ -42,37 +39,36 @@ public class ConditionReference extends Condition { public ConditionReference(String label,String conditionName) { super(label); Validator.ensureNotNull("Name of referenced condition",conditionName); - this.conditionName=conditionName; + this.conditionName = conditionName; setContextName(conditionName); } /** Returns the name of the referenced rule, never null */ public String getConditionName() { return conditionName; } - public void setConditionName(String name) { this.conditionName=name; } + public void setConditionName(String name) { this.conditionName = name; } public boolean doesMatch(RuleEvaluation e) { if (automataLookup) return automataMatch(e); - if (namedCondition==null) - throw new EvaluationException("Condition reference '" + conditionName + - "' not found or not initialized"); + if (namedCondition == null) + throw new EvaluationException("Condition reference '" + conditionName + "' not found or not initialized"); return namedCondition.matches(e); } private boolean automataMatch(RuleEvaluation e) { - FlattenedItem current=e.currentItem(); - if (current==null) return false; + FlattenedItem current = e.currentItem(); + if (current == null) return false; - Object annotation=current.getItem().getAnnotation(conditionName); - if (annotation==null) return false; + Object annotation = current.getItem().getAnnotation(conditionName); + if (annotation == null) return false; if (! (annotation instanceof PhraseMatcher.Phrase)) return false; - PhraseMatcher.Phrase phrase=(PhraseMatcher.Phrase)annotation; + PhraseMatcher.Phrase phrase = (PhraseMatcher.Phrase)annotation; - Choicepoint choicePoint=e.getChoicepoint(this,true); - boolean matches=automataMatchPhrase(phrase,e); + Choicepoint choicePoint = e.getChoicepoint(this,true); + boolean matches = automataMatchPhrase(phrase,e); if (!matches && e.isInNegation()) { // TODO: Temporary hack! Works for single items only e.addMatch(current,null); @@ -84,40 +80,41 @@ public class ConditionReference extends Condition { return matches; } - private boolean automataMatchPhrase(PhraseMatcher.Phrase phrase,RuleEvaluation e) { - for (PhraseMatcher.Phrase.MatchIterator i=phrase.itemIterator(); i.hasNext(); ) { + private boolean automataMatchPhrase(PhraseMatcher.Phrase phrase, RuleEvaluation e) { + for (PhraseMatcher.Phrase.MatchIterator i = phrase.itemIterator(); i.hasNext(); ) { i.next(); - FlattenedItem current=e.currentItem(); - if (current==null) return false; + FlattenedItem current = e.currentItem(); + if (current == null) return false; if (!labelMatches(e.currentItem().getItem(),e)) return false; if (!e.isInNegation()) - e.addMatch(current,i.getReplace()); + e.addMatch(current, i.getReplace()); e.next(); } - if (phrase.getLength()>phrase.getBackedLength()) return false; // The underlying composite item has changed + if (phrase.getLength() > phrase.getBackedLength()) return false; // The underlying composite item has changed return true; } public void makeReferences(RuleBase ruleBase) { - namedCondition=ruleBase.getCondition(conditionName); - if (namedCondition==null) { // Then this may reference some automata value, if we have an automata + namedCondition = ruleBase.getCondition(conditionName); + if (namedCondition == null) { // Then this may reference some automata value, if we have an automata if (ruleBase.usesAutomata()) - automataLookup=true; + automataLookup = true; else throw new RuleBaseException("Referenced condition '" + conditionName + - "' does not exist in " + ruleBase); + "' does not exist in " + ruleBase); } } protected boolean hasOpenChoicepoint(RuleEvaluation e) { - if (namedCondition==null) return false; + if (namedCondition == null) return false; return namedCondition.getCondition().hasOpenChoicepoint(e); } protected boolean isDefaultContextName() { - return getContextName()==null || getContextName().equals(conditionName); + return getContextName() == null || getContextName().equals(conditionName); } + @Override protected String toInnerString() { return "[" + conditionName + "]"; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java index 0d5ce78baf0..af7aab23b85 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java @@ -44,7 +44,7 @@ public class LiteralPhraseProduction extends TermProduction { /** Returns a read only view of the terms produced by this, never null */ public List<String> getTerms() { return Collections.unmodifiableList(terms); } - public void produce(RuleEvaluation e,int offset) { + public void produce(RuleEvaluation e, int offset) { PhraseItem newPhrase = new PhraseItem(); newPhrase.setIndexName(getLabel()); for (String term : terms) @@ -52,13 +52,13 @@ public class LiteralPhraseProduction extends TermProduction { if (replacing) { Match matched = e.getNonreferencedMatch(0); - insertMatch(e, matched, newPhrase, offset); + insertMatch(e, matched, List.of(newPhrase), offset); } else { newPhrase.setWeight(getWeight()); if (e.getTraceLevel() >= 6) e.trace(6, "Adding '" + newPhrase + "'"); - e.addItem(newPhrase, getTermType()); + e.addItems(List.of(newPhrase), getTermType()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java index ed21c9643c3..66507719a11 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java @@ -7,6 +7,8 @@ import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; +import java.util.List; + /** * A literal term produced by a production rule * @@ -63,13 +65,13 @@ public class LiteralTermProduction extends TermProduction { if (replacing) { Match matched = e.getNonreferencedMatch(0); newItem.setWeight(matched.getItem().getWeight()); - insertMatch(e, matched, newItem, offset); + insertMatch(e, matched, List.of(newItem), offset); } else { newItem.setWeight(getWeight()); if (e.getTraceLevel() >= 6) e.trace(6, "Adding '" + newItem + "'"); - e.addItem(newItem, getTermType()); + e.addItems(List.of(newItem), getTermType()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java index a267d274d5a..cbb0b4ab10a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java @@ -33,7 +33,7 @@ public class NamedCondition { e.indentTrace(); } - boolean matches=condition.matches(e); + boolean matches = condition.matches(e); if (e.getTraceLevel() >= 3) { e.unindentTrace(); @@ -50,6 +50,7 @@ public class NamedCondition { * This string representation can always be reparsed to produce an * identical rule to this one. */ + @Override public String toString() { return "[" + conditionName + "] :- " + condition.toString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java index a971020ea90..debc3150959 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java @@ -13,20 +13,20 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; public abstract class Production { /** True to add, false to replace, default true */ - protected boolean replacing=true; + protected boolean replacing = true; /** The (0-base) position of this term in the productions of this rule */ - private int position=0; + private int position = 0; /** The weight (strength) of this production as a percentage (default is 100) */ - private int weight=100; + private int weight = 100; /** Creates a produced template term with no label and the default type */ public Production() { } /** True to replace, false to add, if this production can do both. Default true. */ - public void setReplacing(boolean replacing) { this.replacing=replacing; } + public void setReplacing(boolean replacing) { this.replacing = replacing; } public int getPosition() { return position; } @@ -45,7 +45,7 @@ public abstract class Production { * @param offset the offset position at which to produce this. Offsets are used to produce multiple items * at one position, inserted in the right order. */ - public abstract void produce(RuleEvaluation e,int offset); + public abstract void produce(RuleEvaluation e, int offset); /** * Called to add the references into the condition of this rule made by this production @@ -55,6 +55,7 @@ public abstract class Production { void addMatchReferences(Set<String> matchReferences) { } /** All instances of this produces a parseable string output */ + @Override public final String toString() { return toInnerString() + (getWeight()!=100 ? ("!" + getWeight()) : ""); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java index 46f855ca95d..5d20075cd97 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java @@ -54,6 +54,7 @@ public class ProductionList { } } + @Override public String toString() { StringBuilder buffer = new StringBuilder(); for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java index 20ebc41e57f..b320d10b43d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java @@ -19,18 +19,18 @@ public abstract class ProductionRule { private Condition condition; /** What is produced when this rule is true */ - private ProductionList production=new ProductionList(); + private ProductionList production = new ProductionList(); /** The set of match name Strings which the production part of this rule references */ - private Set<String> matchReferences=new java.util.LinkedHashSet<>(); + private final Set<String> matchReferences = new java.util.LinkedHashSet<>(); /** Sets what must be true for this rule to be true */ - public void setCondition(Condition condition) { this.condition=condition; } + public void setCondition(Condition condition) { this.condition = condition; } public Condition getCondition() { return condition; } /** Sets what is produced when this rule is true */ - public void setProduction(ProductionList production) { this.production=production; } + public void setProduction(ProductionList production) { this.production = production; } public ProductionList getProduction() { return production; } @@ -64,6 +64,7 @@ public abstract class ProductionRule { * This string representation can always be reparsed to produce an * identical rule to this one. */ + @Override public String toString() { return condition.toString() + " " + getSymbol() + " " + production.toString(); } @@ -80,21 +81,27 @@ public abstract class ProductionRule { * This default implementation returns false; */ public boolean isLoop() { - // TODO: There are many more possible loops, we should probably detect - // a few more obvious ones + // TODO: There are many more possible loops, we should probably detect a few more obvious ones if (conditionIsEllipsAndOtherNameSpacesOnly(getCondition())) return true; + if (producesItself()) return true; return false; } private boolean conditionIsEllipsAndOtherNameSpacesOnly(Condition condition) { if (condition instanceof EllipsisCondition) return true; if (! (condition instanceof CompositeCondition)) return false; - for (Iterator<Condition> i=((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { - Condition child= i.next(); - if (child.getNameSpace()==null && conditionIsEllipsAndOtherNameSpacesOnly(child)) + for (Iterator<Condition> i = ((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { + Condition child = i.next(); + if (child.getNameSpace() == null && conditionIsEllipsAndOtherNameSpacesOnly(child)) return true; } return false; } + private boolean producesItself() { + return production.productionList() + .stream() + .anyMatch(p -> (p instanceof ReferenceTermProduction) && ((ReferenceTermProduction)p).producesAll()); + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java index af7abf325e7..00562d3953e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java @@ -1,15 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; import java.util.Set; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.TermType; import com.yahoo.prelude.semantics.engine.EvaluationException; import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.ReferencedMatches; import com.yahoo.prelude.semantics.engine.RuleEvaluation; -import com.yahoo.protect.Validator; /** * A term produced by a production rule which takes its actual term value @@ -19,90 +23,114 @@ import com.yahoo.protect.Validator; */ public class ReferenceTermProduction extends TermProduction { - private String reference; + private final String reference; + private final boolean produceAll; /** * Creates a new produced reference term * * @param reference the label of the condition this should take it's value from */ - public ReferenceTermProduction(String reference) { + public ReferenceTermProduction(String reference, boolean produceAll) { super(); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from * @param termType the type of the term to produce */ - public ReferenceTermProduction(String reference, TermType termType) { + public ReferenceTermProduction(String reference, TermType termType, boolean produceAll) { super(termType); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * * @param label the label of the produced term - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from */ - public ReferenceTermProduction(String label, String reference) { + public ReferenceTermProduction(String label, String reference, boolean produceAll) { super(label); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * * @param label the label of the produced term - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from * @param termType the type of term to produce */ - public ReferenceTermProduction(String label, String reference, TermType termType) { + public ReferenceTermProduction(String label, String reference, TermType termType, boolean produceAll) { super(label, termType); - setReference(reference); - } - - /** The label of the condition this should take its value from, never null */ - public void setReference(String reference) { - Validator.ensureNotNull("reference name of a produced reference term",reference); - this.reference = reference; + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** Returns the label of the condition this should take its value from, never null */ public String getReference() { return reference; } + public boolean producesAll() { return produceAll; } + + @Override void addMatchReferences(Set<String> matchReferences) { matchReferences.add(reference); } - public void produce(RuleEvaluation e, int offset) { + public void produce(RuleEvaluation e, int ignored) { ReferencedMatches referencedMatches = e.getReferencedMatches(reference); if (referencedMatches == null) throw new EvaluationException("Referred match '" + reference + "' not found"); if (replacing) { - replaceMatches(e, referencedMatches); + e.removeMatches(referencedMatches); + } + + var match = referencedMatches.matchIterator().next(); + if (produceAll) { + // produce all terms in the condition + NamedCondition namedCondition = e.getEvaluation().ruleBase().getCondition(referencedMatches.getContextName()); + ChoiceCondition choices = (ChoiceCondition)namedCondition.getCondition(); + List<Item> items = new ArrayList<>(); + for (Iterator<Condition> i = choices.conditionIterator(); i.hasNext();) { + Condition condition = i.next(); + if (condition instanceof TermCondition) { + items.add(match.toItem(getLabel(), ((TermCondition)condition).term())); + } + else if (condition instanceof SequenceCondition) { + PhraseItem phrase = new PhraseItem(getLabel()); + for (var term : ((SequenceCondition)condition).conditions()) + phrase.addItem(match.toItem(getLabel(), ((TermCondition)term).term())); + items.add(phrase); + } + else { + // Until we validate this at construction time + throw new EvaluationException("Could not produce all terms in " + namedCondition + " as it is " + + "not a term or sequence condition"); + } + } + produce(e, match, items, 0); } else { - addMatches(e, referencedMatches); + // produce just the matching term + produce(e, match, List.of(referencedMatches.toItem(getLabel())), 0); } } - public void replaceMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { - Item referencedItem = referencedMatches.toItem(getLabel()); - if (referencedItem == null) return; - e.removeMatches(referencedMatches); - insertMatch(e, referencedMatches.matchIterator().next(), referencedItem, 0); - } - - private void addMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { - Item referencedItem = referencedMatches.toItem(getLabel()); - if (referencedItem == null) return; - e.addItem(referencedItem, getTermType()); + private void produce(RuleEvaluation e, Match match, List<Item> items, int offset) { + if (replacing) + insertMatch(e, match, items, offset); + else + e.addItems(items, getTermType()); } + @Override public String toInnerTermString() { return getLabelString() + "[" + reference + "]"; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java index a2bbf72a53b..eaff66a0bb0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; -import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.semantics.engine.NameSpace; import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.semantics.engine.RuleEvaluation; @@ -49,6 +48,9 @@ public class TermCondition extends Condition { } } + public String term() { return term; } + + @Override public String toInnerString() { return getLabelString() + term; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java index 29e4982ac17..41d15bc9262 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java @@ -7,6 +7,8 @@ import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; +import java.util.List; + /** * A new term produced by a production rule * @@ -59,9 +61,9 @@ public abstract class TermProduction extends Production { * Inserts newItem at the position of this match * TODO: Move to ruleevaluation */ - protected void insertMatch(RuleEvaluation e, Match matched, Item newItem, int offset) { + protected void insertMatch(RuleEvaluation e, Match matched, List<Item> newItems, int offset) { if (getWeight() != 100) - newItem.setWeight(getWeight()); + newItems.forEach(item -> item.setWeight(getWeight())); int insertPosition = matched.getPosition() + offset; // This check is necessary (?) because earlier items may have been removed @@ -71,9 +73,9 @@ public abstract class TermProduction extends Production { insertPosition = matched.getParent().getItemCount(); } - e.insertItem(newItem, matched.getParent(), insertPosition,getTermType()); + e.insertItems(newItems, matched.getParent(), insertPosition, getTermType()); if (e.getTraceLevel() >= 6) - e.trace(6, "Inserted item '" + newItem + "' at position " + insertPosition + " producing " + + e.trace(6, "Inserted items '" + newItems + "' at position " + insertPosition + " producing " + e.getEvaluation().getQuery().getModel().getQueryTree()); } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index c507069b948..9374027504e 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -8,6 +8,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -18,7 +19,6 @@ import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.rendering.Renderer; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; - import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -41,6 +41,8 @@ import java.util.concurrent.Executors; */ public class Execution extends com.yahoo.processing.execution.Execution { + /** @deprecated - applications should define their own summary class instead */ + @Deprecated(since = "7", forRemoval = true) public static final String ATTRIBUTEPREFETCH = "attributeprefetch"; /** @@ -532,18 +534,23 @@ public class Execution extends com.yahoo.processing.execution.Execution { } /** - * Fill hit properties with values from all in-memory attributes. - * This can be done with good performance on many more hits than - * those for which fill is called with the final summary class, so - * if filtering can be done using only in-memory attribute data, - * this method should be preferred over {@link #fill} to get that data for filtering. - * <p> - * Calling this on already filled results has no cost. + * Fill hit properties with values from some in-memory attributes. + * Not all attributes are included, and *which* attributes are + * subject to change depending on what Vespa needs internally. + * + * Applications should prefer to define their own summary class + * with only the in-memory attributes they need, and call + * fill(result, "foo") with the name of their own summary class + * instead of "foo". * + * @deprecated use fill(Result, String) + * + * TODO Remove on Vespa 9. * @param result the result to fill */ + @Deprecated public void fillAttributes(Result result) { - fill(result, ATTRIBUTEPREFETCH); + fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } /** diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index 13a9f9510cd..2dfa6ef3e3a 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -12,6 +12,7 @@ import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass.Fields; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -95,7 +96,7 @@ public class FieldFiller extends Searcher { for (Documentdb db : config.documentdb()) { for (Summaryclass summary : db.summaryclass()) { Set<String> attributes; - if (Execution.ATTRIBUTEPREFETCH.equals(summary.name())) { + if (SORTABLE_ATTRIBUTES_SUMMARY_CLASS.equals(summary.name())) { attributes = new HashSet<>(summary.fields().size()); for (Fields f : summary.fields()) { attributes.add(f.name()); @@ -134,11 +135,11 @@ public class FieldFiller extends Searcher { } if (intersectionOfAttributes.containsAll(summaryFields)) { - if ( ! Execution.ATTRIBUTEPREFETCH.equals(summaryClass)) { - execution.fill(result, Execution.ATTRIBUTEPREFETCH); + if (! SORTABLE_ATTRIBUTES_SUMMARY_CLASS.equals(summaryClass)) { + execution.fill(result, SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } } else { - // Yes, summaryClass may be Execution.ATTRIBUTEPREFETCH here + // Yes, summaryClass may be SORTABLE_ATTRIBUTES_SUMMARY_CLASS here if ( ! summaryDb.hasAll(summaryFields, summaryClass, result.getQuery().getModel().getRestrict())) { execution.fill(result, null); } diff --git a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj index 46117374e59..39ea6435393 100644 --- a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj +++ b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj @@ -69,6 +69,7 @@ TOKEN : <EQUALS: "="> | <EXCLAMATION: "!"> | <INCLUDEDIRECTIVE: "@include"> | + <LANGUAGEDIRECTIVE: "@language"> | <LARGER: ">"> | <LARGEREQUALS: ">="> | <LEFTBRACE: "("> | @@ -86,8 +87,8 @@ TOKEN : <SLASH: "/"> | <SMALLER: "<"> | <SMALLEREQUALS: "<="> | + <STAR: "*"> | <STEMMINGDIRECTIVE: "@stemming"> | - <LANGUAGEDIRECTIVE: "@language"> | <SUPERDIRECTIVE: "@super"> | <IDENTIFIER: (~[ "\u0000"-"\u002f","\u003a"-"\u003f","\u005b"-"\u005d","\u007b"-"\u00a7","\u00a9","\u00ab"-"\u00ae","\u00b0"-"\u00b3","\u00b6"-"\u00b7","\u00b9","\u00bb"-"\u00bf", @@ -312,10 +313,14 @@ NamespaceProduction namespaceProduction() : ReferenceTermProduction referenceTermProduction() : { String reference; + boolean produceAll = false; } { - <LEFTSQUAREBRACKET> reference = referenceIdentifier() <RIGHTSQUAREBRACKET> - { return new ReferenceTermProduction(reference); } + <LEFTSQUAREBRACKET> + reference = referenceIdentifier() + (<STAR> { produceAll = true; })? + <RIGHTSQUAREBRACKET> + { return new ReferenceTermProduction(reference, produceAll); } } LiteralTermProduction literalTermProduction() : diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java index eb69372c22b..17eb4120b84 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java @@ -29,7 +29,7 @@ public class ConditionTestCase { var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); TermCondition term = new TermCondition("foo", linguistics); Query query = new Query("?query=foo"); - assertTrue(term.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(term.matches(new Evaluation(query, null).freshRuleEvaluation())); } @Test @@ -41,11 +41,11 @@ public class ConditionTestCase { sequence.addCondition(term1); sequence.addCondition(term2); Query query = new Query("?query=foo+bar"); - assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query, null).freshRuleEvaluation())); Query query2 = new Query("?query=foo"); - assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2).freshRuleEvaluation())); + assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2, null).freshRuleEvaluation())); Query query3 = new Query("?query=bar"); - assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3).freshRuleEvaluation())); + assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3, null).freshRuleEvaluation())); } @Test @@ -57,11 +57,11 @@ public class ConditionTestCase { choice.addCondition(term1); choice.addCondition(term2); Query query1 = new Query("?query=foo+bar"); - assertTrue(query1 + " matches " + choice, choice.matches(new Evaluation(query1).freshRuleEvaluation())); + assertTrue(query1 + " matches " + choice, choice.matches(new Evaluation(query1, null).freshRuleEvaluation())); Query query2 = new Query("?query=foo"); - assertTrue(query2 + " matches " + choice, choice.matches(new Evaluation(query2).freshRuleEvaluation())); + assertTrue(query2 + " matches " + choice, choice.matches(new Evaluation(query2, null).freshRuleEvaluation())); Query query3 = new Query("?query=bar"); - assertTrue(query3 + " matches " + choice, choice.matches(new Evaluation(query3).freshRuleEvaluation())); + assertTrue(query3 + " matches " + choice, choice.matches(new Evaluation(query3, null).freshRuleEvaluation())); } @Test @@ -75,13 +75,13 @@ public class ConditionTestCase { ProductionRule rule = new ReplacingProductionRule(); rule.setCondition(reference); rule.setProduction(new ProductionList()); - RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); + RuleBase ruleBase = new RuleBase("test"); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); Query query = new Query("?query=foo"); - assertTrue(query + " matches " + reference,reference.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(query + " matches " + reference,reference.matches(new Evaluation(query, null).freshRuleEvaluation())); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java index ca78cae70ea..c398f0ed99e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java @@ -17,8 +17,8 @@ public class MatchOnlyIfNotOnlyTermTestCase extends RuleBaseAbstractTestCase { @Test public void testMatch() { - assertSemantics("RANK (AND justin timberlake) showname:\"saturday night live\"!1000", "justin timberlake snl"); - assertSemantics("RANK (AND justin timberlake) showname:\"saturday night live\"!1000", "justin timberlake saturday night live"); + assertSemantics("RANK showname:\"saturday night live\"!1000 (AND justin timberlake)", "justin timberlake snl"); + assertSemantics("RANK showname:\"saturday night live\"!1000 (AND justin timberlake)", "justin timberlake saturday night live"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java index 3051dd77190..1d6eea8d7de 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java @@ -14,7 +14,7 @@ public class OrPhraseTestCase extends RuleBaseAbstractTestCase { @Test public void testReplacing1() { - assertSemantics("OR (AND new york) title:\"software engineer\"","software engineer new york"); + assertSemantics("OR title:\"software engineer\" (AND new york)","software engineer new york"); assertSemantics("title:\"software engineer\"","software engineer"); // Skip or when there is nothing else } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java index b91e9441a2b..bafb2cb6a73 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java @@ -32,7 +32,7 @@ public class ProductionRuleTestCase { NamedCondition named = new NamedCondition("brand", term); ConditionReference reference = new ConditionReference("brand"); - TermProduction termProduction = new ReferenceTermProduction("brand", "brand"); + TermProduction termProduction = new ReferenceTermProduction("brand", "brand", false); ProductionList productionList = new ProductionList(); productionList.addProduction(termProduction); @@ -41,7 +41,7 @@ public class ProductionRuleTestCase { rule.setProduction(productionList); // To initialize the condition reference... - RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); + RuleBase ruleBase = new RuleBase("test"); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); @@ -49,7 +49,7 @@ public class ProductionRuleTestCase { assertTrue("Brand is referenced", rule.matchReferences().contains("brand")); Query query = new Query("?query=sony"); - RuleEvaluation e = new Evaluation(query).freshRuleEvaluation(); + RuleEvaluation e = new Evaluation(query, null).freshRuleEvaluation(); assertTrue(rule.matches(e)); rule.produce(e); assertEquals("AND brand:sony", query.getModel().getQueryTree().getRoot().toString()); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java index 76b2d3991c1..bee65db4347 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java @@ -128,12 +128,12 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { @Test public void testTypeChange() { - assertSemantics("RANK doors default:typechange","typechange doors"); + assertSemantics("RANK default:typechange doors","typechange doors"); } @Test public void testTypeChangeWithSingularToPluralButNonReplaceWillNotSingularify() { - assertSemantics("RANK door default:typechange","typechange door"); + assertSemantics("RANK default:typechange door","typechange door"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java new file mode 100644 index 00000000000..8f69be2f710 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import org.junit.Test; + +/** + * @author bratseth + */ +public class SynonymTestCase { + + @Test + public void testReplacingBySynonyms() { + var tester = new RuleBaseTester("synonyms.sr"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:foo"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:bar"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:baz"); + tester.assertSemantics("EQUIV index1:word index1:\"a phrase\"", "index1:word"); + tester.assertSemantics("EQUIV index1:word index1:\"a phrase\"", "index1:a index1:phrase"); + tester.assertSemantics("index1:other", "index1:other"); + } + + @Test + public void testAddingSynonyms() { + var tester = new RuleBaseTester("synonyms.sr"); + tester.assertSemantics("EQUIV index2:foo index2:baz index2:bar", "index2:foo"); + tester.assertSemantics("EQUIV index2:bar index2:foo index2:baz", "index2:bar"); + tester.assertSemantics("EQUIV index2:baz index2:foo index2:bar", "index2:baz"); + tester.assertSemantics("EQUIV index2:word index2:\"a phrase\"", "index2:word"); + tester.assertSemantics("EQUIV index2:\"a phrase\" index2:word", "index2:a index2:phrase"); + tester.assertSemantics("index2:other", "index2:other"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr index 9a147887207..32f8e86b59f 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr @@ -5,6 +5,4 @@ equiv1 +> =equiv2 =equiv3; testfield:[test] -> =testfield:e1 =testfield:e2 =testfield:e3; -synonymfield:[test] -> =[test]; - [test] :- foo, bar, baz; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr new file mode 100644 index 00000000000..4220f807733 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr @@ -0,0 +1,11 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +index1:[synonyms1] -> =index1:[synonyms1*]; # Replace by equiv(foo, bar, baz) when the query contains foo, bar or baz +index1:[synonyms2] -> =index1:[synonyms2*]; # with phrase + + +index2:[synonyms1] +> =index2:[synonyms1*]; # Add equiv(foo, bar, baz) when the query contains foo, bar or baz +index2:[synonyms2] +> =index2:[synonyms2*]; # with phrase + +[synonyms1] :- foo, baz, bar; +[synonyms2] :- word, a phrase; diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 5d3a95efc78..0385de5bebf 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -24,7 +24,9 @@ import com.yahoo.search.Searcher; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; -import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS;; +import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; + /** * Test translation of fields and sources in YQL to the associated concepts in Vespa. @@ -52,7 +54,7 @@ public class YqlFieldAndSourceTestCase { mockBackend.addResult(query, result); mockBackend.addSummaryClassByCopy(DEFAULT_SUMMARY_CLASS, Arrays.asList(FIELD1, FIELD2)); - mockBackend.addSummaryClassByCopy(Execution.ATTRIBUTEPREFETCH, Arrays.asList(FIELD2)); + mockBackend.addSummaryClassByCopy(SORTABLE_ATTRIBUTES_SUMMARY_CLASS, Arrays.asList(FIELD2)); mockBackend.addSummaryClassByCopy(THIRD_OPTION, Arrays.asList(FIELD3)); DocumentdbInfoConfig config = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder() @@ -90,7 +92,7 @@ public class YqlFieldAndSourceTestCase { new Fields.Builder().name(FIELD2).type("string"))), new Summaryclass.Builder() .id(1) - .name(Execution.ATTRIBUTEPREFETCH) + .name(SORTABLE_ATTRIBUTES_SUMMARY_CLASS) .fields(Arrays.asList(new Fields.Builder().name(FIELD2).type("string"))), new Summaryclass.Builder() .id(2) @@ -112,7 +114,7 @@ public class YqlFieldAndSourceTestCase { execution.fill(result); assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -123,7 +125,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertTrue(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertTrue(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -134,7 +136,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -145,7 +147,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java index 9d51c5ca9d1..f0b681fd9c9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; -import org.apache.hc.core5.http.ClassicHttpRequest; import java.util.stream.Stream; @@ -41,7 +40,8 @@ public class ConfigServerException extends RuntimeException { PARENT_HOST_NOT_READY, CERTIFICATE_NOT_READY, LOAD_BALANCER_NOT_READY, - INCOMPLETE_RESPONSE + INCOMPLETE_RESPONSE, + CONFIG_NOT_CONVERGED } public static ConfigServerException readException(byte[] body, String context) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 225634634b2..7b1cc1fd84d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -253,6 +253,7 @@ public class InternalStepRunner implements StepRunner { return result; case ACTIVATION_CONFLICT: case APPLICATION_LOCK_FAILURE: + case CONFIG_NOT_CONVERGED: logger.log("Deployment failed with possibly transient error " + e.code() + ", will retry: " + e.getMessage()); return result; @@ -349,14 +350,12 @@ public class InternalStepRunner implements StepRunner { String failureReason = null; - NodeList suspendedTooLong = nodeList - .isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) - .and(nodeList - .not().isStateful() - .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) - ); - if ( ! suspendedTooLong.isEmpty()) { + NodeList suspendedTooLong = nodeList.isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) + .and(nodeList.not().isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) + ); + if ( ! suspendedTooLong.isEmpty() && deployment.get().at().plus(timeouts.statelessNodesDown()).isBefore(controller.clock().instant())) { failureReason = "Some nodes have been suspended for more than the allowed threshold:\n" + suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 2b9e3dd0733..4b38306f905 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -210,7 +210,7 @@ public class Run { return lastVespaLogTimestamp; } - /** Returns the timestamp of the last time no nodes were allowed to be down. */ + /** Returns since when no nodes have been allowed to be down. */ public Optional<Instant> noNodesDownSince() { return noNodesDownSince; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index df670addcd8..b316a620a9b 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -31,8 +31,8 @@ public class ValuesFetcher { private final MetricsConsumers metricsConsumers; public ValuesFetcher(MetricsManager metricsManager, - VespaServices vespaServices, - MetricsConsumers metricsConsumers) { + VespaServices vespaServices, + MetricsConsumers metricsConsumers) { this.metricsManager = metricsManager; this.vespaServices = vespaServices; this.metricsConsumers = metricsConsumers; @@ -47,7 +47,7 @@ public class ValuesFetcher { .collect(Collectors.toList()); } - public MetricsPacket.Builder [] fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { + public MetricsPacket.Builder[] fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); List<MetricsPacket.Builder> builders = metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now(), consumer) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 7f57ec219ae..4d4456920e7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -819,6 +819,7 @@ public class Nodes { private static boolean parkOnDeallocationOf(Node node, Agent agent) { if (node.state() == Node.State.parked) return false; if (agent == Agent.operator) return false; + if (!node.type().isHost() && node.status().wantToDeprovision()) return false; boolean retirementRequestedByOperator = node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire) .map(History.Event::agent) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json index a7a6a06ae0f..50ca2b1ca41 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json @@ -1,7 +1,7 @@ { "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com", "id": "node55", - "state": "parked", + "state": "dirty", "type": "tenant", "hostname": "host55.yahoo.com", "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", @@ -22,7 +22,7 @@ "agent": "system" }, { - "event": "parked", + "event": "deallocated", "at": 123, "agent": "system" } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json index 03df6c8e1dc..2b650bad39b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json @@ -17,8 +17,8 @@ @include(node2.json), @include(docker-node1.json), @include(node1.json), - @include(node5.json), @include(node55.json), + @include(node5.json), @include(dockerhost6.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json index 8835945dc92..55e216f454a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json @@ -17,7 +17,7 @@ @include(node2.json), @include(docker-node1.json), @include(node1.json), - @include(node5.json), - @include(node55.json) + @include(node55.json), + @include(node5.json) ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json index db4d0bb1682..54ff2bc232f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json @@ -52,10 +52,10 @@ "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" }, { - "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" } ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json index 68ff9fedc00..27767be6315 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json @@ -45,6 +45,7 @@ "dirty": { "url": "http://localhost:8080/nodes/v2/state/dirty", "nodes": [ + @include(node55.json) ] }, "failed": { @@ -56,7 +57,6 @@ "parked": { "url": "http://localhost:8080/nodes/v2/state/parked", "nodes": [ - @include(node55.json) ] }, "deprovisioned": { diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index d1a56b8ac8a..5ffb37feb70 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -121,12 +121,6 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { return getInstallVariable("zts_url"); } - @Override - public String zooKeeperSnapshotMethod() { - String vespaZookeeperSnapshotMethod = System.getenv("VESPA_ZOOKEEPER_SNAPSHOT_METHOD"); - return vespaZookeeperSnapshotMethod == null ? "" : vespaZookeeperSnapshotMethod; - } - static ConfigServer[] toConfigServers(String configserversString) { return multiValueParameterStream(configserversString) .map(CloudConfigInstallVariables::toConfigServer) diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index d57ef11f2d3..66955be2325 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -8,6 +8,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.ResourceReference; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Result; @@ -71,22 +72,25 @@ class ClientFeederV3 { this.hostName = HostName.getLocalhost(); } - public boolean timedOut() { + boolean timedOut() { synchronized (monitor) { return Instant.now().isAfter(prevOpsPerSecTime.plusSeconds(6000)) && ongoingRequests.get() == 0; } } - public void kill() { - // No new requests should be sent to this object, but there can be old one, even though this is very unlikely. - while (ongoingRequests.get() > 0) { - try { - ongoingRequests.wait(100); - } catch (InterruptedException e) { - break; + void kill() { + try (ResourceReference ignored = sourceSession.getReference()) { + // No new requests should be sent to this object, but there can be old one, even though this is very unlikely. + while (ongoingRequests.get() > 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + return; + } } + } catch (Exception e) { + log.log(Level.WARNING, "Failed to close reference to source session", e); } - sourceSession.getReference().close(); } private void transferPreviousRepliesToResponse(BlockingQueue<OperationStatus> operations) throws InterruptedException { @@ -98,7 +102,7 @@ class ClientFeederV3 { } } - public HttpResponse handleRequest(HttpRequest request) throws IOException { + HttpResponse handleRequest(HttpRequest request) throws IOException { ongoingRequests.incrementAndGet(); try { FeederSettings feederSettings = new FeederSettings(request); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index 9a1aaa33720..2a774995601 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -144,20 +144,5 @@ public class FeedHandler extends ThreadedHttpRequestHandler { } } - @Override - protected void destroy() { - feedHandlerV3.destroy(); - // We are forking this to avoid that accidental dereferrencing causes any random thread doing destruction. - // This caused a deadlock when the single Messenger thread in MessageBus was the last one referring this - // and started destructing something that required something only the messenger thread could provide. - Thread destroyer = new Thread(() -> { - internalDestroy(); - }); - destroyer.setDaemon(true); - destroyer.start(); - } - - private void internalDestroy() { - super.destroy(); - } + @Override protected void destroy() { feedHandlerV3.destroy(); } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index 620c8400434..95e98f325e1 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.yolean.Exceptions; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -53,8 +52,8 @@ public class FeedHandlerV3 extends ThreadedHttpRequestHandler { docTypeManager = new DocumentTypeManager(documentManagerConfig); this.sessionCache = sessionCache; feedReplyHandler = new FeedReplyReader(metric, metricsHelper); - cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandlerv3.cron")); - cron.scheduleWithFixedDelay(this::removeOldClients, 16, 11, TimeUnit.MINUTES); + cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feed-handler-v3-janitor")); + cron.scheduleWithFixedDelay(this::removeOldClients, 3, 3, TimeUnit.SECONDS); this.metric = metric; } @@ -101,19 +100,18 @@ public class FeedHandlerV3 extends ThreadedHttpRequestHandler { @Override protected void destroy() { - // We are forking this to avoid that accidental dereferrencing causes any random thread doing destruction. + // We are forking this to avoid that accidental de-referencing causes any random thread doing destruction. // This caused a deadlock when the single Messenger thread in MessageBus was the last one referring this // and started destructing something that required something only the messenger thread could provide. Thread destroyer = new Thread(() -> { - super.destroy(); cron.shutdown(); synchronized (monitor) { - for (ClientFeederV3 client : clientFeederByClientId.values()) { - client.kill(); + for (var iterator = clientFeederByClientId.values().iterator(); iterator.hasNext(); ) { + iterator.next().kill(); + iterator.remove(); } - clientFeederByClientId.clear(); } - }); + }, "feed-handler-v3-adhoc-destroyer"); destroyer.setDaemon(true); destroyer.start(); } @@ -142,9 +140,8 @@ public class FeedHandlerV3 extends ThreadedHttpRequestHandler { private void removeOldClients() { synchronized (monitor) { - for (Iterator<Map.Entry<String, ClientFeederV3>> iterator = clientFeederByClientId - .entrySet().iterator(); iterator.hasNext();) { - ClientFeederV3 client = iterator.next().getValue(); + for (var iterator = clientFeederByClientId.values().iterator(); iterator.hasNext(); ) { + ClientFeederV3 client = iterator.next(); if (client.timedOut()) { client.kill(); iterator.remove(); |