diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain |
Publish
Diffstat (limited to 'config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain')
14 files changed, 1260 insertions, 0 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java new file mode 100644 index 00000000000..0dbc7368954 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java @@ -0,0 +1,269 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.collections.CollectionUtil; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec; +import com.yahoo.vespa.model.container.component.Component; + +import java.util.*; + +/** + * Config producer for the FederationSearcher. + * @author tonytv + */ +public class FederationSearcher extends Searcher<FederationSearcherModel> implements FederationConfig.Producer { + + private final Optional<Component> targetSelector; + + /** + * Generates config for a single search chain contained in a target. + */ + private static final class SearchChainConfig { + private final SearchChain searchChain; + //Zero if not applicable + final ComponentId providerId; + final FederationOptions targetOptions; + final List<String> documentTypes; + + SearchChainConfig(SearchChain searchChain, ComponentId providerId, + FederationOptions targetOptions, List<String> documentTypes) { + this.searchChain = searchChain; + this.providerId = providerId; + this.targetOptions = targetOptions; + this.documentTypes = documentTypes; + } + + public FederationConfig.Target.SearchChain.Builder getSearchChainConfig() { + FederationConfig.Target.SearchChain.Builder sB = new FederationConfig.Target.SearchChain.Builder(); + FederationOptions resolvedOptions = targetOptions.inherit(searchChain.federationOptions()); + sB. + searchChainId(searchChain.getGlobalComponentId().stringValue()). + timeoutMillis(resolvedOptions.getTimeoutInMilliseconds()). + requestTimeoutMillis(resolvedOptions.getRequestTimeoutInMilliseconds()). + optional(resolvedOptions.getOptional()). + useByDefault(resolvedOptions.getUseByDefault()). + documentTypes(documentTypes); + if (providerId != null) + sB.providerId(providerId.stringValue()); + return sB; + } + } + + /** + * One or more search chains that are handled as a single group, + * which can be federated to as a single entity. + */ + private static abstract class Target { + final ComponentId id; + final FederationOptions targetOptions; + + public Target(ComponentId id, FederationOptions targetOptions) { + this.id = id; + this.targetOptions = targetOptions; + } + + public FederationConfig.Target.Builder getTargetConfig() { + FederationConfig.Target.Builder tb = new FederationConfig.Target.Builder(); + tb. + id(id.stringValue()). + useByDefault(targetOptions.getUseByDefault()); + getSearchChainsConfig(tb); + return tb; + } + + protected abstract void getSearchChainsConfig(FederationConfig.Target.Builder tb); + } + + private static class SearchChainTarget extends Target { + private final SearchChainConfig searchChainConfig; + + public SearchChainTarget(SearchChain searchChain, + FederationOptions targetOptions) { + super(searchChain.getComponentId(), targetOptions); + searchChainConfig = new SearchChainConfig( + searchChain, + null, + targetOptions, + searchChain.getDocumentTypes()); + } + + @Override + protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) { + tB.searchChain(searchChainConfig.getSearchChainConfig()); + } + } + + private static class SourceGroupTarget extends Target { + private final SearchChainConfig leaderConfig; + private final List<SearchChainConfig> participantsConfig = + new ArrayList<>(); + + public SourceGroupTarget(SourceGroup group, + FederationOptions targetOptions) { + super(group.getComponentId(), applyDefaultSourceGroupOptions(targetOptions)); + + leaderConfig = createConfig(group.leader(), targetOptions); + for (Source participant : group.participants()) { + participantsConfig.add( + createConfig(participant, targetOptions)); + } + } + + private static FederationOptions applyDefaultSourceGroupOptions(FederationOptions targetOptions) { + FederationOptions defaultSourceGroupOption = new FederationOptions().setUseByDefault(true); + return targetOptions.inherit(defaultSourceGroupOption); + } + + private SearchChainConfig createConfig(Source source, + FederationOptions targetOptions) { + return new SearchChainConfig( + source, + source.getParentProvider().getComponentId(), + targetOptions, + source.getDocumentTypes()); + } + + @Override + protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) { + tB.searchChain(leaderConfig.getSearchChainConfig()); + for (SearchChainConfig participant : participantsConfig) { + tB.searchChain(participant.getSearchChainConfig()); + } + } + } + + private static class TargetResolver { + final ComponentRegistry<SearchChain> searchChainRegistry; + final SourceGroupRegistry sourceGroupRegistry; + + /** + * @return true if searchChain.id newer than sourceGroup.id + */ + private boolean newerVersion(SearchChain searchChain, + SourceGroup sourceGroup) { + if (searchChain == null || sourceGroup == null) { + return false; + } else { + return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId()); + } + } + + /** + * @return true if a newer than b + */ + private boolean newerVersion(ComponentId a, ComponentId b) { + return a.compareTo(b) > 0; + } + + + TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, + SourceGroupRegistry sourceGroupRegistry) { + this.searchChainRegistry = searchChainRegistry; + this.sourceGroupRegistry = sourceGroupRegistry; + } + + Target resolve(FederationSearcherModel.TargetSpec specification) { + SearchChain searchChain = searchChainRegistry.getComponent( + specification.sourceSpec); + SourceGroup sourceGroup = sourceGroupRegistry.getComponent( + specification.sourceSpec); + + if (searchChain == null && sourceGroup == null) { + return null; + } else if (sourceGroup == null || + newerVersion(searchChain, sourceGroup)) { + return new SearchChainTarget(searchChain, specification.federationOptions); + } else { + return new SourceGroupTarget(sourceGroup, specification.federationOptions); + } + } + } + + private final Map<ComponentId, Target> resolvedTargets = + new LinkedHashMap<>(); + + public FederationSearcher(FederationSearcherModel searcherModel, Optional<Component> targetSelector) { + super(searcherModel); + this.targetSelector = targetSelector; + + if (targetSelector.isPresent()) + addChild(targetSelector.get()); + } + + @Override + public void getConfig(FederationConfig.Builder builder) { + for (Target target : resolvedTargets.values()) { + builder.target(target.getTargetConfig()); + } + + if (targetSelector.isPresent()) { + builder.targetSelector(targetSelector.get().getGlobalComponentId().stringValue()); + } + } + + @Override + public void initialize() { + initialize(getSearchChains().allChains(), getSearchChains().allSourceGroups()); + } + + void initialize(ComponentRegistry<SearchChain> searchChainRegistry, + SourceGroupRegistry sourceGroupRegistry) { + TargetResolver targetResolver = new TargetResolver( + searchChainRegistry, sourceGroupRegistry); + + addSourceTargets(targetResolver, model.targets); + + if (model.inheritDefaultSources) + addDefaultTargets(targetResolver, searchChainRegistry); + } + + private void addSourceTargets(TargetResolver targetResolver, List<TargetSpec> targets) { + for (TargetSpec targetSpec : targets) { + + Target target = targetResolver.resolve(targetSpec); + if (target == null) { + throw new RuntimeException("Can't find source " + + targetSpec.sourceSpec + + " used as a source for federation '" + + getComponentId() + "'"); + } + + Target duplicate = resolvedTargets.put(target.id, target); + if (duplicate != null && !duplicate.targetOptions.equals(target.targetOptions)) { + throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" + + " to the federation searcher " + getComponentId()); + } + } + } + + + private void addDefaultTargets(TargetResolver targetResolver, ComponentRegistry<SearchChain> searchChainRegistry) { + for (GenericTarget genericTarget : defaultTargets(searchChainRegistry.allComponents())) { + ComponentSpecification specification = genericTarget.getComponentId().toSpecification(); + + //Can't use genericTarget directly, as it might be part of a source group. + Target federationTarget = targetResolver.resolve(new TargetSpec(specification, new FederationOptions())); + //Do not replace manually added sources, as they might have manually configured federation options + if (!resolvedTargets.containsKey(federationTarget.id)) + resolvedTargets.put(federationTarget.id, federationTarget); + } + } + + + private static List<GenericTarget> defaultTargets(Collection<SearchChain> allSearchChains) { + Collection<Provider> providers = + CollectionUtil.filter(allSearchChains, Provider.class); + + List<GenericTarget> defaultTargets = new ArrayList<>(); + for (Provider provider : providers) { + defaultTargets.addAll(provider.defaultFederationTargets()); + } + return defaultTargets; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java new file mode 100644 index 00000000000..9ed62f15244 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +/** + * A search chain that is intended to be used for federation (i.e. providers, sources) + * @author tonytv + */ +abstract public class GenericTarget extends SearchChain { + + private final FederationOptions federationOptions; + + public GenericTarget(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) { + super(specWithoutInnerSearchers); + this.federationOptions = federationOptions; + } + + @Override + public FederationOptions federationOptions() { + FederationOptions defaultOptions = new FederationOptions().setUseByDefault(useByDefault()); + return federationOptions.inherit(defaultOptions); + } + + /** The value for useByDefault in case the user have not specified any **/ + abstract protected boolean useByDefault(); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java new file mode 100644 index 00000000000..ef6c6ef4df6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.cache.QrBinaryCacheConfig; +import com.yahoo.search.cache.QrBinaryCacheRegionConfig; +import com.yahoo.search.federation.ProviderConfig; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.HttpProviderSpec; + +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.search.federation.ProviderConfig.Node; +import static com.yahoo.search.federation.ProviderConfig.Yca; + + +/** + * A provider containing a http searcher. + * @author tonytv + */ +public class HttpProvider extends Provider implements ProviderConfig.Producer, + QrBinaryCacheConfig.Producer, + QrBinaryCacheRegionConfig.Producer { + + private final HttpProviderSpec providerSpec; + + //TODO: For backward compatibility only, eliminate this later + private BinaryScaledAmount cacheSize; + + public double getCacheWeight() { + return providerSpec.cacheWeight; + } + + /** + * TODO: remove, for backward compatibility only. + */ + public void setCacheSize(BinaryScaledAmount cacheSize) { + this.cacheSize = cacheSize; + } + + /* + * Config producer for the contained http searcher.. + */ + + public HttpProvider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions, HttpProviderSpec providerSpec) { + super(specWithoutInnerSearchers, federationOptions); + this.providerSpec = providerSpec; + } + + @Override + public void getConfig(ProviderConfig.Builder builder) { + if (providerSpec.path != null) + builder.path(providerSpec.path); + if (providerSpec.connectionParameters.readTimeout != null) + builder.readTimeout(providerSpec.connectionParameters.readTimeout ); + if (providerSpec.connectionParameters.connectionTimeout != null) + builder.connectionTimeout(providerSpec.connectionParameters.connectionTimeout); + if (providerSpec.connectionParameters.connectionPoolTimeout != null) + builder.connectionPoolTimeout(providerSpec.connectionParameters.connectionPoolTimeout); + if (providerSpec.connectionParameters.retries != null) + builder.retries(providerSpec.connectionParameters.retries); + + builder.node(getNodes(providerSpec.nodes)); + + if (providerSpec.ycaApplicationId != null) { + builder.yca(getYca(providerSpec)); + } + } + + private static Yca.Builder getYca(HttpProviderSpec providerSpec) { + Yca.Builder yca = new Yca.Builder() + .applicationId(providerSpec.ycaApplicationId); + + if (providerSpec.ycaProxy != null) { + yca.useProxy(true); + if (providerSpec.ycaProxy.host != null) { + yca.host(providerSpec.ycaProxy.host) + .port(providerSpec.ycaProxy.port); + } + } + if (providerSpec.ycaCertificateTtl != null) yca.ttl(providerSpec.ycaCertificateTtl); + if (providerSpec.ycaRetryWait != null) yca.ttl(providerSpec.ycaRetryWait); + return yca; + } + + private static List<Node.Builder> getNodes(List<HttpProviderSpec.Node> nodeSpecs) { + ArrayList<Node.Builder> nodes = new ArrayList<>(); + for (HttpProviderSpec.Node node : nodeSpecs) { + nodes.add( + new Node.Builder() + .host(node.host) + .port(node.port)); + } + return nodes; + } + + public int cacheSizeMB() { + return providerSpec.cacheSizeMB != null ? + providerSpec.cacheSizeMB : + (int) cacheSize.as(BinaryPrefix.mega); + } + + @Override + public void getConfig(QrBinaryCacheConfig.Builder builder) { + builder.cache_size(cacheSizeMB()); + } + + @Override + public void getConfig(QrBinaryCacheRegionConfig.Builder builder) { + builder.region_size(cacheSizeMB()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java new file mode 100644 index 00000000000..44f9879230d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.search.searchchain.model.federation.HttpProviderSpec; +import java.util.List; + +/** + +* @author tonytv +*/ +public class HttpProviderSearcher extends Searcher<ChainedComponentModel> { + + + public HttpProviderSearcher(ChainedComponentModel model) { + super(model); + } + + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java new file mode 100644 index 00000000000..1dd4fb478ec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -0,0 +1,188 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.prelude.cluster.QrMonitorConfig; +import com.yahoo.search.config.dispatchprototype.SearchNodesConfig; +import com.yahoo.vespa.config.search.DispatchConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.LocalProviderSpec; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchNode; + +import java.util.*; + +/** + * Config producer for search chain responsible for sending queries to a local cluster. + * + * @author tonytv + */ +public class LocalProvider extends Provider implements + DocumentdbInfoConfig.Producer, + ClusterConfig.Producer, + AttributesConfig.Producer, + QrMonitorConfig.Producer, + RankProfilesConfig.Producer, + SearchNodesConfig.Producer, + DispatchConfig.Producer { + + private final LocalProviderSpec providerSpec; + private volatile AbstractSearchCluster searchCluster; + + + @Override + public void getConfig(ClusterConfig.Builder builder) { + assert (searchCluster != null) : "Null search cluster!"; + builder.clusterId(searchCluster.getClusterIndex()); + builder.clusterName(searchCluster.getClusterName()); + + if (providerSpec.cacheSize != null) + builder.cacheSize(providerSpec.cacheSize); + + if (searchCluster.getVisibilityDelay() != null) + builder.cacheTimeout(convertVisibilityDelay(searchCluster.getVisibilityDelay())); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + searchCluster.getConfig(builder); + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + searchCluster.getConfig(builder); + } + + @Override + public void getConfig(QrMonitorConfig.Builder builder) { + int requestTimeout = federationOptions().getTimeoutInMilliseconds(); + if (requestTimeout != -1) { + builder.requesttimeout(requestTimeout); + } + } + + @Override + public void getConfig(final SearchNodesConfig.Builder builder) { + if (!(searchCluster instanceof IndexedSearchCluster)) { + log.warning("Could not build SearchNodesConfig: Only supported for IndexedSearchCluster, got " + + searchCluster.getClass().getCanonicalName()); + return; + } + final IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster) searchCluster; + for (final SearchNode searchNode : indexedSearchCluster.getSearchNodes()) { + builder.search_node( + new SearchNodesConfig.Search_node.Builder() + .host(searchNode.getHostName()) + .port(searchNode.getDispatchPort())); + } + } + + private void addProviderSearchers(LocalProviderSpec providerSpec) { + for (ChainedComponentModel searcherModel : providerSpec.searcherModels) { + addInnerComponent(new Searcher<>(searcherModel)); + } + } + + @Override + public ChainSpecification getChainSpecification() { + ChainSpecification spec = + super.getChainSpecification(); + return new ChainSpecification(spec.componentId, spec.inheritance, spec.phases(), + disableStemmingIfStreaming(spec.componentReferences)); + } + + //TODO: ugly, restructure this + private Set<ComponentSpecification> disableStemmingIfStreaming(Set<ComponentSpecification> searcherReferences) { + if (!searchCluster.isStreaming()) { + return searcherReferences; + } else { + Set<ComponentSpecification> filteredSearcherReferences = new LinkedHashSet<>(searcherReferences); + filteredSearcherReferences.remove( + toGlobalComponentId( + new ComponentId("com.yahoo.prelude.querytransform.StemmingSearcher")). + toSpecification()); + return filteredSearcherReferences; + } + } + + private ComponentId toGlobalComponentId(ComponentId searcherId) { + return searcherId.nestInNamespace(getComponentId()); + } + + public String getClusterName() { + return providerSpec.clusterName; + } + + public void setSearchCluster(AbstractSearchCluster searchCluster) { + assert (this.searchCluster == null); + this.searchCluster = searchCluster; + } + + public LocalProvider(ChainSpecification specWithoutInnerSearchers, + FederationOptions federationOptions, + LocalProviderSpec providerSpec) { + super(specWithoutInnerSearchers, federationOptions); + addProviderSearchers(providerSpec); + this.providerSpec = providerSpec; + } + + @Override + public List<String> getDocumentTypes() { + List<String> documentTypes = new ArrayList<>(); + + for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) { + documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName()); + } + + return documentTypes; + } + + @Override + public FederationOptions federationOptions() { + Double queryTimeoutInSeconds = searchCluster.getQueryTimeout(); + + return queryTimeoutInSeconds == null ? + super.federationOptions() : + super.federationOptions().inherit( + new FederationOptions().setTimeoutInMilliseconds((int) (queryTimeoutInSeconds * 1000))); + } + + @Override + public void getConfig(DocumentdbInfoConfig.Builder builder) { + searchCluster.getConfig(builder); + } + + /** + * For backward compatibility only, do not use. + */ + public void setCacheSize(Integer cacheSize) { + providerSpec.cacheSize = cacheSize; + } + + // The semantics of visibility delay in search is deactivating caches if the + // delay is less than 1.0, in qrs the cache is deactivated if the delay is 0 + // (or less). 1.0 seems a little arbitrary, so just doing the conversion + // here instead of having two totally independent implementations having to + // follow each other down in the modules. + private static Double convertVisibilityDelay(Double visibilityDelay) { + return (visibilityDelay < 1.0d) ? 0.0d : visibilityDelay; + } + + @Override + public void getConfig(DispatchConfig.Builder builder) { + if (!(searchCluster instanceof IndexedSearchCluster)) { + log.warning("Could not build DispatchConfig: Only supported for IndexedSearchCluster, got " + + searchCluster.getClass().getCanonicalName()); + return; + } + ((IndexedSearchCluster) searchCluster).getConfig(builder); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java new file mode 100644 index 00000000000..8dba2e8b589 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.vespa.model.container.component.ConfigProducerGroup; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Base config producer for search chains that communicate with backends. + * + * @author tonytv + */ +public class Provider extends GenericTarget { + + private ConfigProducerGroup<Source> sources; + + public Provider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) { + super(specWithoutInnerSearchers, federationOptions); + sources = new ConfigProducerGroup<>(this, "source"); + } + + public void addSource(Source source) { + sources.addComponent(source.getComponentId(), source); + } + + public Collection<Source> getSources() { + return sources.getComponents(); + } + + @Override + protected boolean useByDefault() { + return sources.getComponents().isEmpty(); + } + + public Collection<? extends GenericTarget> defaultFederationTargets() { + if (sources.getComponents().isEmpty()) { + return Arrays.asList(this); + } else { + return sources.getComponents(); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java new file mode 100644 index 00000000000..148bab3f84a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.vespa.model.container.component.chain.Chain; + +import java.util.Collections; +import java.util.List; + +/** + * Represents a search chain in the vespa model. + * + * @author tonytv + */ +public class SearchChain extends Chain<Searcher<?>> { + + public SearchChain(ChainSpecification specWithoutInnerSearchers) { + super(specWithoutInnerSearchers); + } + + public FederationOptions federationOptions() { + return new FederationOptions().setUseByDefault(true); + } + + //A list of documents types that this search chain provides results for, empty if unknown + public List<String> getDocumentTypes() { + return Collections.emptyList(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java new file mode 100644 index 00000000000..d13e13b232f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.collections.CollectionUtil; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.chain.Chains; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.LocalClustersCreator; +import com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.VespaSearchChainsCreator; + +import java.util.Collection; +import java.util.Map; + +/** + * Root config producer of the whole search chains model (contains searchchains and searchers). + * + * @author tonytv + */ +public class SearchChains extends Chains<SearchChain> { + + private final SourceGroupRegistry sourceGroups = new SourceGroupRegistry(); + + public SearchChains(AbstractConfigProducer parent, String subId) { + super(parent, subId); + } + + public void initialize(Map<String, ? extends AbstractSearchCluster> searchClustersByName, BinaryScaledAmount totalProviderCacheSize) { + LocalClustersCreator.addDefaultLocalProviders(this, searchClustersByName.keySet()); + VespaSearchChainsCreator.addVespaSearchChains(this); + + validateSourceGroups(); //must be done before initializing searchers since they are used by FederationSearchers. + initializeComponents(searchClustersByName, totalProviderCacheSize); + } + + private void initializeComponents(Map<String, ? extends AbstractSearchCluster> searchClustersByName, + BinaryScaledAmount totalProviderCacheSize) { + setSearchClusterForLocalProvider(searchClustersByName); + setCacheSizeForHttpProviders(totalProviderCacheSize); + initializeComponents(); + } + + private void setCacheSizeForHttpProviders(BinaryScaledAmount totalProviderCacheSize) { + double totalCacheWeight = 0; + for (HttpProvider provider : httpProviders()) { + totalCacheWeight += provider.getCacheWeight(); + } + + final BinaryScaledAmount cacheUnit = totalProviderCacheSize.divide(totalCacheWeight); + for (HttpProvider provider : httpProviders()) { + provider.setCacheSize(cacheUnit.multiply(provider.getCacheWeight())); + } + } + + private void setSearchClusterForLocalProvider(Map<String, ? extends AbstractSearchCluster> clusterIndexByName) { + for (LocalProvider provider : localProviders()) { + AbstractSearchCluster cluster = clusterIndexByName.get(provider.getClusterName()); + if (cluster == null) { + throw new RuntimeException("No searchable content cluster with id '" + provider.getClusterName() + "'"); + } + provider.setSearchCluster(cluster); + } + } + + private void validateSourceGroups() { + for (SourceGroup sourceGroup : sourceGroups.groups()) { + sourceGroup.validate(); + + if (getChainGroup().getComponentMap().containsKey(sourceGroup.getComponentId())) { + throw new RuntimeException( + String.format("Same id used for a source and another search chain/provider: '%s'", + sourceGroup.getComponentId())); + } + } + } + + @Override + public void validate() throws Exception { + validateSourceGroups(); + super.validate(); + } + + public SourceGroupRegistry allSourceGroups() { + return sourceGroups; + } + + public Collection<LocalProvider> localProviders() { + return CollectionUtil.filter(allChains().allComponents(), LocalProvider.class); + } + + + public Collection<HttpProvider> httpProviders() { + return CollectionUtil.filter(allChains().allComponents(), HttpProvider.class); + } + + /* + * If searchChain is a provider, its sources must already have been attached. + */ + @Override + public void add(SearchChain searchChain) { + assert !(searchChain instanceof Source); + + super.add(searchChain); + + if (searchChain instanceof Provider) { + sourceGroups.addSources((Provider)searchChain); + } + } + + @Override + public ComponentRegistry<SearchChain> allChains() { + ComponentRegistry<SearchChain> allChains = new ComponentRegistry<>(); + for (SearchChain chain : getChainGroup().getComponents()) { + allChains.register(chain.getId(), chain); + if (chain instanceof Provider) + addSources(allChains, (Provider)chain); + } + allChains.freeze(); + return allChains; + } + + private void addSources(ComponentRegistry<SearchChain> chains, Provider provider) { + for (Source source : provider.getSources()) { + chains.register(source.getId(), source); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java new file mode 100644 index 00000000000..653be591be3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; + +/** + * @author gjoranv + * @author tonytv + */ +public class Searcher<T extends ChainedComponentModel> extends ChainedComponent<T> { + + public Searcher(T model) { + super(model); + } + + protected SearchChains getSearchChains() { + AbstractConfigProducer ancestor = getParent(); + while (!(ancestor instanceof SearchChains)) { + ancestor = ancestor.getParent(); + } + return (SearchChains)ancestor; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java new file mode 100644 index 00000000000..fe839b904d2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +import java.util.Arrays; + + +/** + * Config producer for source, which is contained in a provider. + * + * @author tonytv + */ +public class Source extends GenericTarget { + + //Each source group must have exactly one leader, and an arbitrary number of participants + public enum GroupOption { + leader, + participant + } + + public final GroupOption groupOption; + + public Source(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions, + GroupOption groupOption) { + super(specWithoutInnerSearchers, federationOptions); + this.groupOption = groupOption; + } + + @Override + public FederationOptions federationOptions() { + return super.federationOptions().inherit(getParentProvider().federationOptions()); + } + + @Override + protected boolean useByDefault() { + return false; + } + + public Provider getParentProvider() { + AbstractConfigProducer parent = getParent(); + while (!(parent instanceof Provider)) { + parent = parent.getParent(); + } + return (Provider)parent; + } + + @Override + public ChainSpecification getChainSpecification() { + return super.getChainSpecification().addInherits( + Arrays.asList(getParentProvider().getComponentId().toSpecification())); + } + + public ComponentId getGlobalComponentId() { + return getComponentId().nestInNamespace( + getParentProvider().getComponentId()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java new file mode 100644 index 00000000000..799bba45b04 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * A set of sources with the same name, + * each associated with a different provider, + * that fills the same role. + * @author tonytv + */ +final class SourceGroup { + private final ComponentId id; + private Source leader; + private final Set<Source> participants = + new LinkedHashSet<>(); + + private void setLeader(Source leader) { + assert (validMember(leader)); + + if (this.leader != null) { + throw new IllegalStateException( + "There can not be two default providers for the source " + + id); + } + + this.leader = leader; + } + + private void addParticipant(Source source) { + assert (validMember(source)); + assert (!source.equals(leader)); + + if (!participants.add(source)) { + throw new RuntimeException("Source added twice to the same group " + + source); + } + } + + private boolean validMember(Source leader) { + return leader.getComponentId().equals(id); + } + + public ComponentId getComponentId() { + return id; + } + + public SourceGroup(ComponentId id) { + this.id = id; + } + + public void add(Source source) { + assert source.getComponentId().equals(getComponentId()): + "Ids differ: " + source.getComponentId() + " -- " + getComponentId(); + + if (Source.GroupOption.leader == source.groupOption) { + setLeader(source); + } else { + addParticipant(source); + } + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Source id: ").append(id).append("\n"). + append("Leader provider: ").append( + leader.getParentProvider().getComponentId()).append("\n"). + append("Participants:"); + + for (Source participant : participants) { + builder.append("\n").append(" Provider: ").append( + participant.getParentProvider().getComponentId()); + } + return builder.toString(); + } + + public Source leader() { + return leader; + } + + public Collection<Source> participants() { + return Collections.unmodifiableCollection(participants); + } + + public void validate() { + if (leader == null) + throw new IllegalStateException("Missing leader for the source " + getComponentId() + + ". One of the sources must use the attribute id instead of idref."); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java new file mode 100644 index 00000000000..fda962402d1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.model.ComponentAdaptor; +import com.yahoo.component.provider.ComponentRegistry; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + + +/** + * Owns all the source groups in the search chains model. + * @author tonytv + */ +class SourceGroupRegistry { + private final ComponentRegistry<ComponentAdaptor<SourceGroup>> sourceGroups + = new ComponentRegistry<>(); + + private void add(Source source) { + getGroup(source.getComponentId()).add(source); + } + + private SourceGroup getGroup(ComponentId sourceId) { + ComponentAdaptor<SourceGroup> group = + sourceGroups.getComponent(sourceId); + if (group == null) { + group = new ComponentAdaptor<>(sourceId, + new SourceGroup(sourceId)); + sourceGroups.register(group.getId(), group); + } + return group.model; + } + + void addSources(Provider provider) { + for (Source source : provider.getSources()) { + add(source); + } + } + + public Collection<SourceGroup> groups() { + List<SourceGroup> result = new ArrayList<>(); + for (ComponentAdaptor<SourceGroup> group : + sourceGroups.allComponents()) { + result.add(group.model); + } + return result; + } + + public SourceGroup getComponent(ComponentSpecification spec) { + ComponentAdaptor<SourceGroup> result = sourceGroups.getComponent(spec); + return (result != null)? + result.model : + null; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java new file mode 100644 index 00000000000..43f4a28ff30 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.LocalProviderSpec; +import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Adds default search chains for all local clusters not mentioned explicitly + * @author tonytv + */ +public class LocalClustersCreator { + static ChainSpecification emptySearchChainSpecification(String componentName) { + return new ChainSpecification( + new ComponentId(componentName), + VespaSearchChainsCreator.inheritsVespaPhases(), //TODO: refactor + Collections.<Phase>emptyList(), + Collections.<ComponentSpecification>emptySet()); + } + + static LocalProvider createDefaultLocalProvider(String clusterName) { + return new LocalProvider(emptySearchChainSpecification(clusterName), new FederationOptions(), + new LocalProviderSpec(clusterName, null)); + } + + static Set<String> presentClusters(SearchChains searchChains) { + Set<String> presentClusters = new LinkedHashSet<>(); + for (LocalProvider provider : searchChains.localProviders()) { + presentClusters.add(provider.getClusterName()); + } + return presentClusters; + } + + public static void addDefaultLocalProviders(SearchChains searchChains, Set<String> clusterNames) { + Set<String> missingClusters = new LinkedHashSet<>(clusterNames); + missingClusters.removeAll(presentClusters(searchChains)); + + for (String clusterName : missingClusters) { + searchChains.add(createDefaultLocalProvider(clusterName)); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java new file mode 100644 index 00000000000..ed5fd3b759e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.search.searchchain.PhaseNames; +import com.yahoo.search.searchchain.model.VespaSearchers; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.search.searchchain.*; + +import java.util.*; + +/** + * Creates the search chains vespaPhases, vespa and native. + * + * <p>TODO: refactor</p> + * @author tonytv + */ +public class VespaSearchChainsCreator { + private static class PhasesCreator { + private static Set<String> set(String successor) { + return successor == null ? null : new LinkedHashSet<>(Arrays.asList(successor)); + } + + private static String lastElement(String[] phases) { + return phases[phases.length - 1]; + } + + private static Phase createPhase(String phase, String before) { + return new Phase(phase, set(before), null); + } + + public static Collection<Phase> linearPhases(String... phases) { + List<Phase> result = new ArrayList<>(); + + for (int i=0; i < phases.length - 1; ++i) { + result.add( + createPhase(phases[i], phases[i+1])); + } + + if (phases.length > 0) { + result.add( + createPhase(lastElement(phases), null)); + } + + return result; + } + } + + private static Set<ComponentSpecification> noSearcherReferences() { + return Collections.emptySet(); + } + + private static Collection<Phase> noPhases() { + return Collections.emptySet(); + } + + private static ChainSpecification.Inheritance inherits(ComponentId chainId) { + Set<ComponentSpecification> inheritsSet = new LinkedHashSet<>(); + inheritsSet.add(chainId.toSpecification()); + return new ChainSpecification.Inheritance(inheritsSet, null); + } + + static ChainSpecification.Inheritance inheritsVespaPhases() { + return inherits(vespaPhasesSpecification().componentId); + } + + private static void addInnerSearchers(SearchChain searchChain, Collection<ChainedComponentModel> searcherModels) { + for (ChainedComponentModel searcherModel : searcherModels) { + searchChain.addInnerComponent(createSearcher(searcherModel)); + } + } + + private static Searcher<? extends ChainedComponentModel> createSearcher(ChainedComponentModel searcherModel) { + if (searcherModel instanceof FederationSearcherModel) { + return new FederationSearcher((FederationSearcherModel) searcherModel, Optional.<Component>empty()); + } else { + return new Searcher<>(searcherModel); + } + } + + + private static ChainSpecification nativeSearchChainSpecification() { + return new ChainSpecification( + new ComponentId("native"), + inheritsVespaPhases(), + noPhases(), + noSearcherReferences()); + } + + private static ChainSpecification vespaSearchChainSpecification() { + return new ChainSpecification( + new ComponentId("vespa"), + inherits(nativeSearchChainSpecification().componentId), + noPhases(), + noSearcherReferences()); + } + + + private static ChainSpecification vespaPhasesSpecification() { + return new ChainSpecification( + new ComponentId("vespaPhases"), + new ChainSpecification.Inheritance(null, null), + PhasesCreator.linearPhases( + PhaseNames.RAW_QUERY, + PhaseNames.TRANSFORMED_QUERY, + PhaseNames.BLENDED_RESULT, + PhaseNames.UNBLENDED_RESULT, + PhaseNames.BACKEND), + noSearcherReferences()); + } + + private static SearchChain createVespaPhases() { + return new SearchChain(vespaPhasesSpecification()); + } + + private static SearchChain createNative() { + SearchChain nativeChain = new SearchChain(nativeSearchChainSpecification()); + addInnerSearchers(nativeChain, VespaSearchers.nativeSearcherModels); + return nativeChain; + } + + private static SearchChain createVespa() { + SearchChain vespaChain = new SearchChain(vespaSearchChainSpecification()); + addInnerSearchers(vespaChain, VespaSearchers.vespaSearcherModels); + return vespaChain; + } + + public static void addVespaSearchChains(SearchChains searchChains) { + searchChains.add( + createVespaPhases()); + searchChains.add( + createNative()); + searchChains.add( + createVespa()); + } +} |