diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2024-02-14 11:11:33 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2024-02-14 11:17:04 +0100 |
commit | bfcd156d488df1d8ce82bd575cf637328a214693 (patch) | |
tree | 003f6a48328f258dea9debf4758256d7f781c276 /container-search | |
parent | 267cc7f36883721d9c61634cb87dbee76b0c3e3d (diff) |
- Add a resolver for virtual sources.
- Move sourceref tests out of separate test package.
Diffstat (limited to 'container-search')
-rw-r--r-- | container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java | 18 | ||||
-rw-r--r-- | container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java | 4 | ||||
-rw-r--r-- | container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java | 65 | ||||
-rw-r--r-- | container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java (renamed from container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java) | 10 | ||||
-rw-r--r-- | container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java (renamed from container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java) | 12 | ||||
-rw-r--r-- | container-search/src/test/java/com/yahoo/search/federation/sourceref/VirtualSourceResolverTestCase.java | 29 |
6 files changed, 115 insertions, 23 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index e92fd692cac..1f8f8757ebc 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -25,6 +25,7 @@ import com.yahoo.search.federation.sourceref.SingleTarget; import com.yahoo.search.federation.sourceref.SourceRefResolver; import com.yahoo.search.federation.sourceref.SourcesTarget; import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.federation.sourceref.VirtualSourceResolver; import com.yahoo.search.query.Properties; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; @@ -77,25 +78,28 @@ public class FederationSearcher extends ForkingSearcher { private final SearchChainResolver searchChainResolver; private final SourceRefResolver sourceRefResolver; + private final VirtualSourceResolver virtualSourceResolver; private final TargetSelector<?> targetSelector; private final Clock clock = Clock.systemUTC(); @Inject public FederationSearcher(FederationConfig config, ComponentRegistry<TargetSelector> targetSelectors) { - this(createResolver(config), resolveSelector(config.targetSelector(), targetSelectors)); + this(createResolver(config), VirtualSourceResolver.of(config), resolveSelector(config.targetSelector(), targetSelectors)); } // for testing public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver) { - this(searchChainResolver, null); + this(searchChainResolver, VirtualSourceResolver.of(), null); } private FederationSearcher(SearchChainResolver searchChainResolver, + VirtualSourceResolver virtualSourceResolver, TargetSelector targetSelector) { this.searchChainResolver = searchChainResolver; sourceRefResolver = new SourceRefResolver(searchChainResolver); this.targetSelector = targetSelector; + this.virtualSourceResolver = virtualSourceResolver; } private static TargetSelector resolveSelector(String selectorId, @@ -370,9 +374,9 @@ public class FederationSearcher extends ForkingSearcher { else { // fill timed out: Remove these hits as they are incomplete and may cause a race when accessed later result.hits().addError(futureFilledResult.getSecond().createTimeoutError()); for (Iterator<Hit> i = futureFilledResult.getFirst().hits().unorderedDeepIterator(); i.hasNext(); ) { - // Note that some of these hits may be filled, but as the fill thread may still be working on them - // and we do not synchronize with it we need to discard all - Hit removed = result.hits().remove(i.next().getId()); + // Note that some of these hits may be filled, but as the fill thread may still be working on them, + // and we do not synchronize with it, we need to discard all. + result.hits().remove(i.next().getId()); } } } @@ -427,8 +431,10 @@ public class FederationSearcher extends ForkingSearcher { resolveSources(sources, properties, indexFacts); } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sources, Properties properties, IndexFacts indexFacts) { + + private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sourcesInQuery, Properties properties, IndexFacts indexFacts) { Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>(); + Set<String> sources = virtualSourceResolver.resolve(sourcesInQuery); for (String source : sources) { try { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java index a2bc12ddbd0..df91b968750 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java @@ -56,10 +56,6 @@ public class SearchChainResolver { } }; - public Builder addSearchChain(ComponentId searchChainId) { - return addSearchChain(searchChainId, List.of()); - } - public Builder addSearchChain(ComponentId searchChainId, FederationOptions federationOptions) { return addSearchChain(searchChainId, federationOptions, List.of()); } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java new file mode 100644 index 00000000000..fc07d12d429 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/VirtualSourceResolver.java @@ -0,0 +1,65 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref; + +import com.yahoo.search.federation.FederationConfig; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Multiple sources like contentcluster.schema1, contencluster.schema2 needs to respond + * when source is the prefix contentcluster. This is done by generating map from virtual source + * to the fully qualified ones, and resolving from there. + * + * @author baldersheim + */ +public class VirtualSourceResolver { + private final Map<String, Set<String>> virtualSources; + private VirtualSourceResolver(Map<String, Set<String>> virtualSources) { + this.virtualSources = virtualSources; + } + public static VirtualSourceResolver of() { + return new VirtualSourceResolver(Map.of()); + } + public static VirtualSourceResolver of(Set<String> targets) { + return new VirtualSourceResolver(createVirtualSources(targets)); + } + private static Map<String, Set<String>> createVirtualSources(Set<String> targets) { + Set<String> virtualSources = targets.stream() + .filter(id -> id.contains(".")) + .map(id -> id.substring(0, id.indexOf('.'))) + .collect(Collectors.toUnmodifiableSet()); + if (virtualSources.isEmpty()) return Map.of(); + Map<String, Set<String>> virtualSourceMap = new HashMap<>(); + for (String virtualSource : virtualSources) { + String prefix = virtualSource + "."; + Set<String> sources = targets.stream() + .filter(id -> id.startsWith(prefix)) + .collect(Collectors.toUnmodifiableSet()); + virtualSourceMap.put(virtualSource, sources); + } + return virtualSourceMap; + } + public static VirtualSourceResolver of(FederationConfig config) { + return of(config.target().stream().map(FederationConfig.Target::id).collect(Collectors.toUnmodifiableSet())); + } + public Set<String> resolve(Set<String> sourcesInQuery) { + boolean hasMapping = sourcesInQuery.stream().anyMatch(virtualSources::containsKey); + if (hasMapping) { + Set<String> resolved = new HashSet<>(); + for (String source : sourcesInQuery) { + var subSources = virtualSources.get(source); + if (subSources != null) { + resolved.addAll(subSources); + } else { + resolved.add(source); + } + } + return resolved; + } + return sourcesInQuery; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java index d575be603c1..d9046075f38 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java @@ -1,14 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref.test; +package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.processing.request.properties.PropertyMap; import com.yahoo.processing.request.Properties; -import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; -import com.yahoo.search.federation.sourceref.SearchChainResolver; -import com.yahoo.search.federation.sourceref.Target; -import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; @@ -16,7 +12,9 @@ import java.util.Collections; import java.util.Iterator; import java.util.SortedSet; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.fail; /** * @author Tony Vaagenes diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java index 1b3baebac6f..9badd4a1ab6 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java @@ -1,14 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref.test; +package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; -import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; -import com.yahoo.search.federation.sourceref.SearchChainResolver; -import com.yahoo.search.federation.sourceref.SourceRefResolver; -import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; @@ -18,8 +14,10 @@ import java.util.List; import java.util.Set; import java.util.TreeMap; -import static com.yahoo.search.federation.sourceref.test.SearchChainResolverTestCase.emptySourceToProviderMap; -import static org.junit.jupiter.api.Assertions.*; +import static com.yahoo.search.federation.sourceref.SearchChainResolverTestCase.emptySourceToProviderMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Test for SourceRefResolver. diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/VirtualSourceResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/VirtualSourceResolverTestCase.java new file mode 100644 index 00000000000..bf6605a43b2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/VirtualSourceResolverTestCase.java @@ -0,0 +1,29 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref; + +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * Test of VirtualSourceResolver + * + * @author baldersheim + */ +public class VirtualSourceResolverTestCase { + @Test + void testThatOriginalIsReturnedIfNoMapping() { + var input = Set.of("a","b", "b.c"); + assertSame(input, VirtualSourceResolver.of().resolve(input)); + assertSame(input, VirtualSourceResolver.of(Set.of("x.a","x.b")).resolve(input)); + } + @Test + void testResolution() { + var input = Set.of("a","b", "b.c"); + assertEquals(Set.of("a.x", "a.y", "b.c", "b.x"), + VirtualSourceResolver.of(Set.of("a.x","a.y", "b.x")).resolve(input)); + } +} |