diff options
Diffstat (limited to 'container-search')
9 files changed, 151 insertions, 97 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 8d01d92fac5..7f100df4e2c 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -6034,6 +6034,7 @@ "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile, com.yahoo.language.process.Embedder)", "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile, java.util.Map)", + "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile, java.util.Map, ai.vespa.cloud.ZoneInfo)", "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", @@ -6729,7 +6730,7 @@ "public" ], "methods": [ - "public void <init>(java.util.Map, ai.vespa.cloud.ZoneInfo)", + "public void <init>(java.util.Map)", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", "public java.util.Map listProperties(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)" @@ -8230,7 +8231,6 @@ "methods": [ "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.schema.SchemaInfo, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public com.yahoo.search.searchchain.Execution newExecution(com.yahoo.component.chain.Chain)", "public com.yahoo.search.searchchain.Execution newExecution(java.lang.String)", "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()", diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index b6deee61b81..ecd6e302ccc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -42,22 +42,6 @@ public class Pong { this.error = error; } - /** - * @deprecated do not use. Additional errors are ignored. - */ - @Deprecated - public void addError(ErrorMessage error) { } - - /** - * @deprecated use error() instead - */ - @Deprecated - public ErrorMessage getError(int i) { - if (i > 1) throw new IllegalArgumentException("No error at position " + i); - if (i == 0 && error.isEmpty()) throw new IllegalArgumentException("No error at position " + i); - return error.get(); - } - public Optional<ErrorMessage> error() { return error; } /** Returns the number of active documents in the backend responding in this Pong, if available */ @@ -66,27 +50,6 @@ public class Pong { /** Returns true if the pinged node is currently blocking write operations due to being full */ public boolean isBlockingWrites() { return isBlockingWrites; } - /** - * Returns Optional.empty() - * - * @return empty - * @deprecated do not use. There is always one pong per node. - */ - @Deprecated - public Optional<Integer> activeNodes() { - return Optional.empty(); - } - - /** - * Returns a list containing 0 or 1 errors - * - * @deprecated use error() instead - */ - @Deprecated - public List<ErrorMessage> getErrors() { - return error.stream().collect(Collectors.toList()); - } - /** Returns whether there is an error or not */ public boolean badResponse() { return error.isPresent(); } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index b50a7ccc2db..48885e4b3da 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -365,7 +365,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { startTime = httpRequest.getJDiscRequest().creationTime(TimeUnit.MILLISECONDS); if (queryProfile != null) { // Move all request parameters to the query profile - Properties queryProfileProperties = new QueryProfileProperties(queryProfile, embedders); + Properties queryProfileProperties = new QueryProfileProperties(queryProfile, embedders, zoneInfo); properties().chain(queryProfileProperties); setPropertiesFromRequestMap(requestMap, properties(), true); @@ -373,11 +373,11 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { properties().chain(new RankProfileInputProperties(schemaInfo, this, embedders)) .chain(new QueryProperties(this, queryProfile.getRegistry(), embedders)) .chain(new ModelObjectMap()) - .chain(new RequestContextProperties(requestMap, zoneInfo)) + .chain(new RequestContextProperties(requestMap)) .chain(queryProfileProperties) .chain(new DefaultProperties()); - // Pass the values from the query profile which maps through a field in the Query object model + // Pass values from the query profile which maps to a field in the Query object model // through the property chain to cause those values to be set in the Query object model with // the right types according to query profiles setFieldsFrom(queryProfileProperties, requestMap); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index abd23c1822d..8c1a0ac1d25 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile; +import ai.vespa.cloud.ZoneInfo; import com.yahoo.collections.Pair; import com.yahoo.language.process.Embedder; import com.yahoo.processing.IllegalInputException; @@ -32,6 +33,7 @@ public class QueryProfileProperties extends Properties { private final CompiledQueryProfile profile; private final Map<String, Embedder> embedders; + private final ZoneInfo zoneInfo; // Note: The priority order is: values has precedence over references @@ -46,18 +48,26 @@ public class QueryProfileProperties extends Properties { private List<Pair<CompoundName, CompiledQueryProfile>> references = null; public QueryProfileProperties(CompiledQueryProfile profile) { - this(profile, Embedder.throwsOnUse.asMap()); + this(profile, Embedder.throwsOnUse.asMap(), ZoneInfo.defaultInfo()); } + @Deprecated // TODO: Remove on Vespa 9 public QueryProfileProperties(CompiledQueryProfile profile, Embedder embedder) { - this(profile, Map.of(Embedder.defaultEmbedderId, embedder)); + this(profile, Map.of(Embedder.defaultEmbedderId, embedder), ZoneInfo.defaultInfo()); } /** Creates an instance from a profile, throws an exception if the given profile is null */ + @Deprecated // TODO: Remove on Vespa 9 public QueryProfileProperties(CompiledQueryProfile profile, Map<String, Embedder> embedders) { + this(profile, embedders, ZoneInfo.defaultInfo()); + } + + /** Creates an instance from a profile, throws an exception if the given profile is null */ + public QueryProfileProperties(CompiledQueryProfile profile, Map<String, Embedder> embedders, ZoneInfo zoneInfo) { Validator.ensureNotNull("The profile wrapped by this cannot be null", profile); this.profile = profile; this.embedders = embedders; + this.zoneInfo = zoneInfo; } /** Returns the query profile backing this, or null if none */ @@ -67,6 +77,7 @@ public class QueryProfileProperties extends Properties { @Override public Object get(CompoundName name, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { + context = contextWithZoneInfo(context); name = unalias(name, context); if (values != null && values.containsKey(name)) return values.get(name); // Returns this value, even if null @@ -92,11 +103,13 @@ public class QueryProfileProperties extends Properties { */ @Override public void set(CompoundName name, Object value, Map<String, String> context) { + context = contextWithZoneInfo(context); setOrCheckSettable(name, value, context, true); } @Override public void requireSettable(CompoundName name, Object value, Map<String, String> context) { + context = contextWithZoneInfo(context); setOrCheckSettable(name, value, context, false); } @@ -210,6 +223,8 @@ public class QueryProfileProperties extends Properties { @Override public Map<String, Object> listProperties(CompoundName path, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { + context = contextWithZoneInfo(context); + path = unalias(path, context); if (context == null) context = Collections.emptyMap(); @@ -257,7 +272,7 @@ public class QueryProfileProperties extends Properties { return properties; } - public boolean isComplete(StringBuilder firstMissingName, Map<String,String> context) { + public boolean isComplete(StringBuilder firstMissingName, Map<String, String> context) { // Are all types reachable from this complete? if ( ! reachableTypesAreComplete(CompoundName.empty, profile, firstMissingName, context)) return false; @@ -272,6 +287,16 @@ public class QueryProfileProperties extends Properties { return true; } + private Map<String, String> contextWithZoneInfo(Map<String, String> context) { + if (zoneInfo == ZoneInfo.defaultInfo()) return context; + + Map<String, String> contextWithZoneInfo = context == null ? new HashMap<>() : new HashMap<>(context); + contextWithZoneInfo.putIfAbsent("environment", zoneInfo.zone().environment().name()); + contextWithZoneInfo.putIfAbsent("region", zoneInfo.zone().region()); + contextWithZoneInfo.putIfAbsent("instance", zoneInfo.application().instance()); + return Collections.unmodifiableMap(contextWithZoneInfo); + } + private boolean reachableTypesAreComplete(CompoundName prefix, CompiledQueryProfile profile, StringBuilder firstMissingName, Map<String,String> context) { for (Map.Entry<CompoundName, DimensionalValue<QueryProfileType>> typeEntry : profile.getTypes().entrySet()) { QueryProfileType type = typeEntry.getValue().get(context); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java index 46aafcb11e3..ad212d05780 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java @@ -21,17 +21,8 @@ public class RequestContextProperties extends Properties { private final Map<String, String> context; - public RequestContextProperties(Map<String, String> properties, ZoneInfo zoneInfo) { - if (zoneInfo == ZoneInfo.defaultInfo()) { - context = properties; - } - else { - Map<String, String> context = new HashMap<>(properties); - context.putIfAbsent("environment", zoneInfo.zone().environment().name()); - context.putIfAbsent("region", zoneInfo.zone().region()); - context.putIfAbsent("instance", zoneInfo.application().instance()); - this.context = Collections.unmodifiableMap(context); - } + public RequestContextProperties(Map<String, String> properties) { + this.context = Collections.unmodifiableMap(properties); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 430a397e6b9..58353cc5907 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -119,18 +119,18 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final String GROUPING_VALUE = "value"; private static final String VESPA_HIDDEN_FIELD_PREFIX = "$"; - private final JsonFactory generatorFactory; + private static final JsonFactory generatorFactory = createGeneratorFactory(); - private JsonGenerator generator; - private FieldConsumer fieldConsumer; - private Deque<Integer> renderedChildren; + private volatile JsonGenerator generator; + private volatile FieldConsumer fieldConsumer; + private volatile Deque<Integer> renderedChildren; static class FieldConsumerSettings { - boolean debugRendering = false; - boolean jsonDeepMaps = true; - boolean jsonWsets = true; - boolean jsonMapsAll = true; - boolean jsonWsetsAll = false; - boolean tensorShortForm = false; + volatile boolean debugRendering = false; + volatile boolean jsonDeepMaps = true; + volatile boolean jsonWsets = true; + volatile boolean jsonMapsAll = true; + volatile boolean jsonWsetsAll = false; + volatile boolean tensorShortForm = false; boolean convertDeep() { return (jsonDeepMaps || jsonWsets); } void init() { this.debugRendering = false; @@ -155,9 +155,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { this.tensorShortForm = q.getPresentation().getTensorShortForm(); } } - private final FieldConsumerSettings fieldConsumerSettings = new FieldConsumerSettings(); - private LongSupplier timeSource; - private OutputStream stream; + private volatile FieldConsumerSettings fieldConsumerSettings; + private volatile LongSupplier timeSource; + private volatile OutputStream stream; public JsonRenderer() { this(null); @@ -169,17 +169,18 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { */ public JsonRenderer(Executor executor) { super(executor); - generatorFactory = new JsonFactory(); - generatorFactory.setCodec(createJsonCodec()); } - private static ObjectMapper createJsonCodec() { - return new ObjectMapper().disable(FLUSH_AFTER_WRITE_VALUE); + private static JsonFactory createGeneratorFactory() { + JsonFactory factory = new JsonFactory(); + factory.setCodec(new ObjectMapper().disable(FLUSH_AFTER_WRITE_VALUE)); + return factory; } @Override public void init() { super.init(); + fieldConsumerSettings = new FieldConsumerSettings(); fieldConsumerSettings.init(); setGenerator(null, fieldConsumerSettings); renderedChildren = null; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index e99dafef886..bfc4219eabc 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -91,19 +91,6 @@ public class ExecutionFactory extends AbstractComponent { this.executor = executor != null ? executor : Executors.newSingleThreadExecutor(); } - /** @deprecated pass SchemaInfoConfig */ - @Deprecated - public ExecutionFactory(ChainsConfig chainsConfig, - IndexInfoConfig indexInfo, - QrSearchersConfig clusters, - ComponentRegistry<Searcher> searchers, - SpecialtokensConfig specialTokens, - Linguistics linguistics, - ComponentRegistry<Renderer> renderers, - Executor executor) { - this(chainsConfig, indexInfo, SchemaInfo.empty(), clusters, searchers, specialTokens, linguistics, renderers, executor); - } - private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, ChainsConfig chainsConfig) { SearchChainRegistry searchChainRegistry = new SearchChainRegistry(searchers); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java index 3542e1413eb..3cf2949f33c 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -1359,33 +1359,90 @@ public class QueryProfileVariantsTestCase { CompiledQueryProfileRegistry cRegistry = registry.compile(); CompiledQueryProfile cTest = cRegistry.findQueryProfile("test"); - assertValueForZone("default", ZoneInfo.defaultInfo(), cTest); + assertValueForZone("default", ZoneInfo.defaultInfo(), null, cTest); assertValueForZone("prod-region1-instance1", new ZoneInfo(new ApplicationId("tenant1", "application1", "instance1"), new Zone(Environment.prod, "region1")), + null, cTest); assertValueForZone("prod-instance2", new ZoneInfo(new ApplicationId("tenant2", "application2", "instance2"), new Zone(Environment.prod, "region1")), + null, cTest); assertValueForZone("prod-region3", new ZoneInfo(new ApplicationId("tenant3", "application3", "instance3"), new Zone(Environment.prod, "region3")), + null, cTest); assertValueForZone("dev", new ZoneInfo(new ApplicationId("tenant4", "application4", "instance4"), new Zone(Environment.dev, "region4")), + null, cTest); } - private void assertValueForZone(String expected, ZoneInfo zoneInfo, CompiledQueryProfile cTest) { - assertEquals(expected, - new Query.Builder().setQueryProfile(cTest).setZoneInfo(zoneInfo).build().properties().get("value")); + @Test + public void testZoneInfoInContextWithUnoverridability() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("test"); + profile.setDimensions(new String[] { "instance", "environment", "region" }); + profile.set("value", "default", registry); + profile.set("value", "prod-beta", + toMap("environment=prod", "instance=beta"), + registry); + profile.setOverridable("value", false, toMap("environment=prod", "instance=beta")); + registry.register(profile); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cTest = cRegistry.findQueryProfile("test"); + + assertValueForZone("prod-beta", + new ZoneInfo(new ApplicationId("tenant1", "application1", "beta"), + new Zone(Environment.prod, "region1")), + "fromRequest", + cTest); + } + + + private void assertValueForZone(String expected, ZoneInfo zoneInfo, String requestValue, CompiledQueryProfile cTest) { + var builder = new Query.Builder().setQueryProfile(cTest).setZoneInfo(zoneInfo); + if (requestValue != null) + builder.setRequestMap(Map.of("value", requestValue)); + assertEquals(expected, builder.build().properties().get("value")); + } + + @Test + public void testZoneInfoInContextSettingNativeProperty() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("test"); + profile.setDimensions(new String[] { "instance", "environment", "region" }); + profile.set("timeout", "0.3", + toMap("environment=prod", "instance=beta"), + registry); + registry.register(profile); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cTest = cRegistry.findQueryProfile("test"); + + assertTimeoutForZone(300, + new ZoneInfo(new ApplicationId("tenant1", "application1", "beta"), + new Zone(Environment.prod, "region1")), + null, + cTest); + } + + private void assertTimeoutForZone(int expected, ZoneInfo zoneInfo, String requestValue, CompiledQueryProfile cTest) { + var builder = new Query.Builder().setQueryProfile(cTest).setZoneInfo(zoneInfo); + if (requestValue != null) + builder.setRequestMap(Map.of("timeout", requestValue)); + assertEquals(expected, builder.build().getTimeout()); } private void assertGet(String expectedValue, String parameter, String[] dimensionValues, QueryProfile profile, CompiledQueryProfile cprofile) { - Map<String,String> context=toMap(profile,dimensionValues); - assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'",expectedValue,cprofile.get(parameter,context)); + Map<String, String> context = toMap(profile,dimensionValues); + assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'", + expectedValue, cprofile.get(parameter,context)); } public static Map<String,String> toMap(QueryProfile profile, String[] dimensionValues) { diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 570b1d006de..83d04c4cc6d 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; +import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.QrSearchersConfig; import com.yahoo.data.access.simple.Value; import com.yahoo.data.access.slime.SlimeAdapter; @@ -55,7 +56,10 @@ import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.trace.TraceNode; +import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -69,6 +73,10 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -87,20 +95,42 @@ public class JsonRendererTestCase { @Rule public Timeout globalTimeout = Timeout.seconds(300); - private final JsonRenderer originalRenderer; + private static ThreadPoolExecutor executor; + private static JsonRenderer blueprint; private JsonRenderer renderer; - public JsonRendererTestCase() { - originalRenderer = new JsonRenderer(); + @BeforeClass + public static void createExecutorAndBlueprint() { + ThreadFactory threadFactory = ThreadFactoryFactory.getThreadFactory("test-rendering"); + executor = new ThreadPoolExecutor(4, 4, 1L, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), threadFactory); + executor.prestartAllCoreThreads(); + blueprint = new JsonRenderer(executor); } @Before - public void setUp() throws Exception { + public void createClone() { // Use the shared renderer as a prototype object, as specified in the API contract - renderer = (JsonRenderer) originalRenderer.clone(); + renderer = (JsonRenderer) blueprint.clone(); renderer.init(); } + @After + public void deconstructClone() { + renderer.deconstruct(); + renderer = null; + } + + @AfterClass + public static void deconstructBlueprintAndExecutor() throws InterruptedException { + blueprint.deconstruct(); + blueprint = null; + executor.shutdown(); + if (!executor.awaitTermination(1, TimeUnit.MINUTES)) { + throw new RuntimeException("Failed to shutdown executor"); + } + executor = null; + } + @Test public void testDocumentId() throws IOException, InterruptedException, ExecutionException { String expected = "{" |