diff options
73 files changed, 346 insertions, 287 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index ec51ba73257..4f937b23429 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -61,8 +61,8 @@ public interface ModelContext { // TODO: Only needed for LbServicesProducerTest default boolean useDedicatedNodeForLogserver() { return true; } - // TODO Revisit in May or June 2020 - boolean useAdaptiveDispatch(); + // TODO Remove when 7.225 is last + default boolean useAdaptiveDispatch() { return true; } default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 035c16b70ba..b06022ba714 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -37,7 +37,6 @@ public class TestProperties implements ModelContext.Properties { private Zone zone; private Set<ContainerEndpoint> endpoints = Collections.emptySet(); private boolean useDedicatedNodeForLogserver = false; - private boolean useAdaptiveDispatch = false; private boolean useDistributorBtreeDb = false; private boolean useThreePhaseUpdates = false; private double defaultTermwiseLimit = 1.0; @@ -60,7 +59,6 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean isBootstrap() { return false; } @Override public boolean isFirstTimeDeployment() { return false; } - @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -123,11 +121,6 @@ public class TestProperties implements ModelContext.Properties { return this; } - public TestProperties setUseAdaptiveDispatch(boolean useAdaptiveDispatch) { - this.useAdaptiveDispatch = useAdaptiveDispatch; - return this; - } - public TestProperties setMultitenant(boolean multitenant) { this.multitenant = multitenant; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 4c4b1ca7f82..8002f89a2e7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -5,10 +5,12 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.ContainerSubsystem; +import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.search.AbstractSearchCluster; @@ -34,6 +36,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> SemanticRulesConfig.Producer, PageTemplatesConfig.Producer { + public static final String QUERY_PROFILE_REGISTRY_CLASS = CompiledQueryProfileRegistry.class.getName(); + private ApplicationContainerCluster owningCluster; private final List<AbstractSearchCluster> searchClusters = new LinkedList<>(); private final Options options; @@ -46,6 +50,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> super(chains); this.owningCluster = cluster; this.options = options; + + owningCluster.addComponent(new SimpleComponent(QUERY_PROFILE_REGISTRY_CLASS)); } public void connectSearchClusters(Map<String, AbstractSearchCluster> searchClusters) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 1f980949738..5e5976c4b9c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -52,7 +52,6 @@ public class IndexedSearchCluster extends SearchCluster private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; - private final boolean useAdaptiveDispatch; private List<SearchNode> searchNodes = new ArrayList<>(); /** @@ -69,7 +68,6 @@ public class IndexedSearchCluster extends SearchCluster super(parent, clusterName, index); unionCfg = new UnionConfiguration(this, documentDbs); rootDispatch = new DispatchGroup(this); - useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch(); } @Override @@ -305,8 +303,6 @@ public class IndexedSearchCluster extends SearchCluster nodeBuilder.port(node.getRpcPort()); builder.node(nodeBuilder); } - if (useAdaptiveDispatch) - builder.distributionPolicy(DistributionPolicy.ADAPTIVE); if (tuning.dispatch.getTopkProbability() != null) { builder.topKProbability(tuning.dispatch.getTopkProbability()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index 47f37a75fd0..a2d13439f4e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.component.ComponentId; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; @@ -16,6 +17,7 @@ import org.w3c.dom.Element; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static com.yahoo.test.Matchers.hasItemWithMethod; +import static com.yahoo.vespa.model.container.search.ContainerSearch.QUERY_PROFILE_REGISTRY_CLASS; import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_BINDING; import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS; import static org.hamcrest.CoreMatchers.is; @@ -127,6 +129,14 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { assertThat(chainsConfig().chains(), hasItemWithMethod("vespa", "id")); } + @Test + public void query_profiles_registry_component_is_added() { + createClusterWithOnlyDefaultChains(); + ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); + var queryProfileRegistryId = ComponentId.fromString(QUERY_PROFILE_REGISTRY_CLASS); + assertTrue(cluster.getComponentsMap().containsKey(queryProfileRegistryId)); + } + private void createClusterWithOnlyDefaultChains() { Element containerElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 5633e1a5eec..5b3c42df869 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -310,14 +310,14 @@ public class ContentClusterTest extends ContentBaseTest { StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); model.getConfig(builder, "bar/distributor/0"); StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); - assertEquals(false, config.inlinebucketsplitting()); + assertFalse(config.inlinebucketsplitting()); } { StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); model.getConfig(builder, "bar/storage/0"); StorFilestorConfig config = new StorFilestorConfig(builder); - assertEquals(false, config.enable_multibit_split_optimalization()); + assertFalse(config.enable_multibit_split_optimalization()); } } @@ -343,7 +343,7 @@ public class ContentClusterTest extends ContentBaseTest { List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); try{ new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); - assertTrue("Deploying without redundancy should fail", false); + fail("Deploying without redundancy should fail"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage(), e.getMessage().contains("missing required element \"redundancy\"")); } @@ -773,7 +773,7 @@ public class ContentClusterTest extends ContentBaseTest { " <documents/>" + " <engine>" + " <proton>" + - " <flush-on-shutdown>" + Boolean.toString(flushOnShutdown) + "</flush-on-shutdown>" + + " <flush-on-shutdown>" + flushOnShutdown + "</flush-on-shutdown>" + " </proton>" + " </engine>" + " <group>" + @@ -944,27 +944,6 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(0.9999, cfg.topKProbability(), 0.0); } - private void verifyRoundRobinPropertiesControl(boolean useAdaptiveDispatch) { - VespaModel model = createEnd2EndOneNode(new TestProperties().setUseAdaptiveDispatch(useAdaptiveDispatch)); - - ContentCluster cc = model.getContentClusters().get("storage"); - DispatchConfig.Builder builder = new DispatchConfig.Builder(); - cc.getSearch().getConfig(builder); - - DispatchConfig cfg = new DispatchConfig(builder); - if (useAdaptiveDispatch) { - assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, cfg.distributionPolicy()); - } else { - assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, cfg.distributionPolicy()); - } - } - - @Test - public void default_dispatch_controlled_by_properties() { - verifyRoundRobinPropertiesControl(false); - verifyRoundRobinPropertiesControl(true); - } - @Test public void default_topKprobability_controlled_by_properties() { verifyTopKProbabilityPropertiesControl(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index b61159e2a4c..7c93b4ef02b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -65,7 +65,7 @@ public class ClusterTest { assertEquals(0.23, config.minWaitAfterCoverageFactor(), DELTA); assertEquals(0.58, config.maxWaitAfterCoverageFactor(), DELTA); assertEquals(2, config.searchableCopies()); - assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); + assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); } @Test @@ -74,7 +74,7 @@ public class ClusterTest { "", joinLines( "<max-hits-per-partition>77</max-hits-per-partition>", - "<dispatch-policy>adaptive</dispatch-policy>", + "<dispatch-policy>round-robin</dispatch-policy>", "<min-group-coverage>13</min-group-coverage>", "<min-active-docs-coverage>93</min-active-docs-coverage>", "<top-k-probability>0.777</top-k-probability>"), @@ -85,7 +85,7 @@ public class ClusterTest { assertEquals(2, config.searchableCopies()); assertEquals(93.0, config.minActivedocsPercentage(), DELTA); assertEquals(13.0, config.minGroupCoverage(), DELTA); - assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); + assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); assertEquals(77, config.maxHitsPerNode()); assertEquals(0.777, config.topKProbability(), DELTA); } @@ -98,7 +98,7 @@ public class ClusterTest { cluster.getSearch().getConfig(builder); DispatchConfig config = new DispatchConfig(builder); assertEquals(2, config.searchableCopies()); - assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); + assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); assertEquals(0, config.maxNodesDownPerGroup()); assertEquals(1.0, config.maxWaitAfterCoverageFactor(), DELTA); assertEquals(0, config.minWaitAfterCoverageFactor(), DELTA); diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index a73032583a5..aa40c317d75 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -14,7 +14,7 @@ minGroupCoverage double default=100 maxNodesDownPerGroup int default=0 # Distribution policy for group selection -distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ROUNDROBIN +distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ADAPTIVE ## Maximum number of hits that will be requested from a single node ## in this dataset. If not set, there is no limit. Using this option diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 9152d87c4ba..5c325ffb752 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -146,7 +146,6 @@ public class ModelContextImpl implements ModelContext { private final Set<ContainerEndpoint> endpoints; private final boolean isBootstrap; private final boolean isFirstTimeDeployment; - private final boolean useAdaptiveDispatch; private final boolean useDistributorBtreeDb; private final boolean useThreePhaseUpdates; private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; @@ -183,8 +182,6 @@ public class ModelContextImpl implements ModelContext { this.endpoints = endpoints; this.isBootstrap = isBootstrap; this.isFirstTimeDeployment = isFirstTimeDeployment; - this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.endpointCertificateSecrets = endpointCertificateSecrets; defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); @@ -240,9 +237,6 @@ public class ModelContextImpl implements ModelContext { public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } @Override - public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } - - @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 286adbd256d..5aee711b379 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -63,7 +63,7 @@ public class SessionCreateHandler extends SessionHandler { return createResponse(request, tenantName, deployLog, sessionId); } - private static ApplicationId getFromApplicationId(HttpRequest request) { + static ApplicationId getFromApplicationId(HttpRequest request) { String from = request.getProperty("from"); if (from == null || "".equals(from)) { throw new BadRequestException("Parameter 'from' has illegal value '" + from + "'"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index a95a02180ca..ae7d5a48955 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -355,7 +355,7 @@ public class ApplicationRepositoryTest { } @Test - public void deletesApplicationRoles() throws IOException { + public void deletesApplicationRoles() { var tenant = tenantRepository.getTenant(tenant1); var applicationId = applicationId(tenant1); var prepareParams = new PrepareParams.Builder().applicationId(applicationId) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 6f883d7e7c0..158b8ea55d2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -86,7 +86,6 @@ public class ModelContextImplTest { assertEquals(new Version(7), context.modelVespaVersion()); assertEquals(new Version(8), context.wantedNodeVespaVersion()); assertEquals(1.0, context.properties().defaultTermwiseLimit(), 0.0); - assertFalse(context.properties().useAdaptiveDispatch()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 5ea2ce5266b..9a1c476117d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -1,17 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStreamTest; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -37,6 +34,7 @@ import static com.yahoo.jdisc.Response.Status.OK; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static com.yahoo.jdisc.http.HttpRequest.Method.POST; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -57,7 +55,6 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { public File testApp = new File("src/test/apps/app"); private LocalSessionRepo localSessionRepo; - private TenantApplications applicationRepo; private TenantRepository tenantRepository; private MockSessionFactory sessionFactory; @@ -67,14 +64,12 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Before public void setupRepo() { - applicationRepo = TenantApplications.create(componentRegistry, new MockReloadHandler(), tenant); localSessionRepo = new LocalSessionRepo(tenant, componentRegistry); tenantRepository = new TenantRepository(componentRegistry, false); sessionFactory = new MockSessionFactory(); TenantBuilder tenantBuilder = TenantBuilder.create(componentRegistry, tenant) .withSessionFactory(sessionFactory) - .withLocalSessionRepo(localSessionRepo) - .withApplicationRepo(applicationRepo); + .withLocalSessionRepo(localSessionRepo); tenantRepository.addTenant(tenantBuilder); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; createdMessage = " for tenant '" + tenant + "' created.\""; @@ -103,19 +98,6 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Request contains no Content-Type header"); } - private void assertFromParameter(String expected, String from) throws IOException { - HttpRequest request = post(Collections.singletonMap("from", from)); - sessionFactory.applicationPackage = testApp; - HttpResponse response = createHandler().handle(request); - assertNotNull(response); - assertThat(response.getStatus(), is(OK)); - assertTrue(sessionFactory.createFromCalled); - assertThat(SessionHandlerTest.getRenderedString(response), - is("{\"log\":[]" + tenantMessage + ",\"session-id\":\"" + expected + "\",\"prepared\":\"http://" + hostname + ":" + port + pathPrefix + - expected + "/prepared\",\"content\":\"http://" + hostname + ":" + port + pathPrefix + - expected + "/content/\",\"message\":\"Session " + expected + createdMessage + "}")); - } - private void assertIllegalFromParameter(String fromValue) throws IOException { File outFile = CompressedApplicationInputStreamTest.createTarFile(); HttpRequest request = post(outFile, postHeaders, Collections.singletonMap("from", fromValue)); @@ -182,24 +164,11 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Test public void require_that_application_urls_can_be_given_as_from_parameter() throws Exception { - localSessionRepo.addSession(new SessionHandlerTest.MockSession(2, FilesApplicationPackage.fromFile(testApp))); - ApplicationId fooId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("foo") - .instanceName("quux") - .build(); - applicationRepo.createApplication(fooId); - applicationRepo.createPutTransaction(fooId, 2).commit(); - assertFromParameter("3", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux"); - localSessionRepo.addSession(new SessionHandlerTest.MockSession(5, FilesApplicationPackage.fromFile(testApp))); - ApplicationId bioId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("foobio") - .instanceName("quux") - .build(); - applicationRepo.createApplication(bioId); - applicationRepo.createPutTransaction(bioId, 5).commit(); - assertFromParameter("6", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foobio/environment/staging/region/baz/instance/quux"); + ApplicationId applicationId = ApplicationId.from(tenant.value(), "foo", "quux"); + HttpRequest request = post(Collections.singletonMap( + "from", + "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux")); + assertEquals(applicationId, SessionCreateHandler.getFromApplicationId(request)); } @Test diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 51fee99a743..2b4424654a2 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -4191,6 +4191,7 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.search.searchchain.ExecutionFactory, java.util.Optional)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.container.core.ContainerHttpConfig)", @@ -6063,8 +6064,9 @@ ], "methods": [ "public void <init>()", + "public void <init>(com.yahoo.search.query.profile.config.QueryProfilesConfig)", "public void <init>(com.yahoo.search.query.profile.types.QueryProfileTypeRegistry)", - "public void register(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", + "public final void register(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", "public com.yahoo.search.query.profile.types.QueryProfileTypeRegistry getTypeRegistry()", "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile findQueryProfile(java.lang.String)" ], diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 338add37213..56dfc700ca7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -190,7 +190,7 @@ public class FastHit extends Hit { /** * Returns values for the features listed in - * <a href="https://docs.vespa.ai/documentation/reference/search-definitions-reference.html#summary-features">summary-features</a> + * <a href="https://docs.vespa.ai/documentation/reference/schema-reference.html#summary-features">summary-features</a> * in the rank profile specified in the query producing this. */ public FeatureData features() { diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index c8386f3c75c..c658d404adb 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -121,6 +121,28 @@ public class SearchHandler extends LoggingRequestHandler { Metric metric, Executor executor, AccessLog accessLog, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig containerHttpConfig, + ExecutionFactory executionFactory) { + this(statistics, + metric, + executor, + accessLog, + queryProfileRegistry, + executionFactory, + containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), + containerHttpConfig.hostResponseHeaderKey().equals("") ? + Optional.empty() : Optional.of(containerHttpConfig.hostResponseHeaderKey())); + } + + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 + public SearchHandler(Statistics statistics, + Metric metric, + Executor executor, + AccessLog accessLog, QueryProfilesConfig queryProfileConfig, ContainerHttpConfig containerHttpConfig, ExecutionFactory executionFactory) { @@ -132,7 +154,7 @@ public class SearchHandler extends LoggingRequestHandler { executionFactory, containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? - Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); + Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); } public SearchHandler(Statistics statistics, diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index 7ab05d0cd1e..744c6eb6933 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -1,9 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.compiled; +import com.google.inject.Inject; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileCompiler; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; /** @@ -18,6 +23,15 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer private final QueryProfileTypeRegistry typeRegistry; + @Inject + public CompiledQueryProfileRegistry(QueryProfilesConfig config) { + QueryProfileRegistry registry = QueryProfileConfigurer.createFromConfig(config); + typeRegistry = registry.getTypeRegistry(); + for (QueryProfile inputProfile : registry.allComponents()) { + register(QueryProfileCompiler.compile(inputProfile, this)); + } + } + /** Creates a compiled query profile registry with no types */ public CompiledQueryProfileRegistry() { this(QueryProfileTypeRegistry.emptyFrozen()); @@ -28,7 +42,7 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer } /** Registers a type by its id */ - public void register(CompiledQueryProfile profile) { + public final void register(CompiledQueryProfile profile) { super.register(profile.getId(), profile); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index cafba58662e..4afb6186c60 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -120,6 +120,7 @@ public class MockSearchCluster extends SearchCluster { builder.minGroupCoverage(99.0); builder.maxNodesDownPerGroup(0); builder.minSearchCoverage(minSearchCoverage); + builder.distributionPolicy(DispatchConfig.DistributionPolicy.Enum.ROUNDROBIN); if (minSearchCoverage < 100.0) { builder.minWaitAfterCoverageFactor(0); builder.maxWaitAfterCoverageFactor(0.5); diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg index 96843d78aae..915da8dc037 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg @@ -1,4 +1,4 @@ -handler[7] +handler[8] handler[0].id com.yahoo.search.handler.SearchHandler handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningHandler handler[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningAsyncHandler @@ -6,3 +6,4 @@ handler[3].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingHandle handler[4].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingAsyncHandler handler[5].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingHandler handler[6].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingAsyncHandler +handler[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java new file mode 100644 index 00000000000..39d4fec2716 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java @@ -0,0 +1,28 @@ +package com.yahoo.search.query.profile.compiled; + +import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author gjoranv + */ +public class CompiledQueryProfileRegistryTest { + + @Test + public void registry_can_be_created_from_config() { + var config = new QueryProfilesConfig.Builder() + .queryprofile(new QueryProfilesConfig.Queryprofile.Builder() + .id("profile1") + .property(new QueryProfilesConfig.Queryprofile.Property.Builder() + .name("hits") + .value("5"))) + .build(); + + var registry = new CompiledQueryProfileRegistry(config); + var profile1 = registry.findQueryProfile("profile1"); + assertEquals("5", profile1.get("hits")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java index 819cd3cdfd4..a1fba8e07f1 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java @@ -1,27 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.config.test; -import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.io.IOUtils; import com.yahoo.search.Query; import com.yahoo.search.query.profile.QueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.query.profile.config.QueryProfilesConfig.Queryprofile; import com.yahoo.search.test.QueryTestCase; -import com.yahoo.vespa.config.ConfigPayload; -import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; -import java.io.File; -import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; + /** * @author bratseth */ diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg index a047ae1cb73..04dcbb22d5d 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg @@ -10,3 +10,4 @@ components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileInt components[4].id com.yahoo.search.handler.SearchHandler components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack components[6].id com.yahoo.search.searchchain.ExecutionFactory +components[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg index ef9d4490a77..b5bc450ec17 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg @@ -10,3 +10,4 @@ components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileInt components[4].id com.yahoo.search.handler.SearchHandler components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack components[6].id com.yahoo.search.searchchain.ExecutionFactory +components[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg index ad20005e7ad..53811bdf536 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg @@ -1,2 +1,3 @@ -handler[1] +handler[2] handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg index ad20005e7ad..53811bdf536 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg @@ -1,2 +1,3 @@ -handler[1] +handler[2] handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg index 8a985f92d10..e1d5c418c6a 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg @@ -22,3 +22,4 @@ components[8].classId com.yahoo.search.searchchain.config.test.twosearchers.Mult components[8].bundle twosearchers components[9].id com.yahoo.search.handler.SearchHandler components[10].id com.yahoo.container.handler.config.HandlersConfigurerDi$RegistriesHack +components[11].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index 7b78f90bc56..51ed93bb6ee 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -60,7 +60,7 @@ import static com.yahoo.document.json.JsonSerializationHelper.*; /** * The DocumentUpdateJsonSerializer utility class is used to serialize a DocumentUpdate instance using the JSON format described in - * <a href="http://docs.vespa.ai/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> + * <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> * * @see #serialize(com.yahoo.document.DocumentUpdate) * @author Vegard Sjonfjell diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index 769e31818e6..ffc276fc94c 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -14,7 +14,7 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.*; /** * Reads the tensor format defined at - * See <a href="http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor">http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor</a> + * See <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html">https://docs.vespa.ai/documentation/reference/document-json-format.html</a> * * @author geirst * @author bratseth diff --git a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp index 5176a4b3e94..1de56e605c9 100644 --- a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp +++ b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp @@ -275,6 +275,7 @@ TEST_F("compile sequentially, then run all conformance tests", test::EvalSpec()) auto t0 = steady_clock::now(); f1.each_case(test); auto t1 = steady_clock::now(); + CompileCache::wait_pending(); auto t2 = steady_clock::now(); test.verify(); auto t3 = steady_clock::now(); @@ -295,7 +296,7 @@ TEST_F("compile concurrently (8 threads), then run all conformance tests", test: auto t0 = steady_clock::now(); f1.each_case(test); auto t1 = steady_clock::now(); - executor.sync(); + CompileCache::wait_pending(); auto t2 = steady_clock::now(); test.verify(); auto t3 = steady_clock::now(); diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp index 9a262f6dca5..4aa18d3bb65 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp @@ -77,6 +77,27 @@ CompileCache::compile(const Function &function, PassParams pass_params) return token; } +void +CompileCache::wait_pending() +{ + std::vector<Token::UP> pending; + { + std::lock_guard<std::mutex> guard(_lock); + for (auto entry = _cached.begin(); entry != _cached.end(); ++entry) { + if (entry->second.compiled_function.get() == nullptr) { + ++(entry->second.num_refs); + pending.push_back(std::make_unique<Token>(entry, Token::ctor_tag())); + } + } + } + { + for (const auto &token: pending) { + const CompiledFunction &fun = token->get(); + (void) fun; + } + } +} + size_t CompileCache::num_cached() { diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.h b/eval/src/vespa/eval/eval/llvm/compile_cache.h index aaadec772a5..09b5b2060f5 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.h +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.h @@ -84,6 +84,7 @@ public: }; static Token::UP compile(const Function &function, PassParams pass_params); + static void wait_pending(); static ExecutorBinding::UP bind(Executor &executor) { return std::make_unique<ExecutorBinding>(executor, ExecutorBinding::ctor_tag()); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index d2e85a1dae3..61205911848 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -109,12 +109,6 @@ public class Flags { "Takes effect on restart of Docker container", NODE_TYPE, APPLICATION_ID, HOSTNAME); - public static final UnboundBooleanFlag USE_ADAPTIVE_DISPATCH = defineFeatureFlag( - "use-adaptive-dispatch", false, - "Should adaptive dispatch be used over round robin", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag REBOOT_INTERVAL_IN_DAYS = defineIntFlag( "reboot-interval-in-days", 30, "No reboots are scheduled 0x-1x reboot intervals after the previous reboot, while reboot is " + @@ -180,9 +174,9 @@ public class Flags { "Takes effect on next host-admin tick.", HOSTNAME); - public static final UnboundStringFlag ZOOKEEPER_SERVER_MAJOR_MINOR_VERSION = defineStringFlag( - "zookeeper-server-version", "3.5", - "The version of ZooKeeper server to use (major.minor, not full version)", + public static final UnboundStringFlag ZOOKEEPER_SERVER_VERSION = defineStringFlag( + "zookeeper-server-version", "3.5.6", + "ZooKeeper server version, a jar file zookeeper-server-<ZOOKEEPER_SERVER_VERSION>-jar-with-dependencies.jar must exist", "Takes effect on restart of Docker container", NODE_TYPE, APPLICATION_ID, HOSTNAME); diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index e855bb015f8..bd8c77bc9cc 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -175,18 +175,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>abi-check-plugin</artifactId> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <systemPropertyVariables> - <jdisc.logger.level>INFO</jdisc.logger.level> - <java.io.tmpdir>${project.build.directory}</java.io.tmpdir> - </systemPropertyVariables> - <trimStackTrace>false</trimStackTrace> - </configuration> - </plugin> </plugins> </build> </project> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index ce696903a53..6097688df72 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -142,6 +142,11 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { .collect(collectingAndThen(Collectors.toList(), NodeList::copyOf)); } + public NodeList group(int index) { + return matching(n -> ( n.allocation().isPresent() && + n.allocation().get().membership().cluster().group().equals(Optional.of(ClusterSpec.Group.from(index))))); + } + /** Returns the parent node of the given child node */ public Optional<Node> parentOf(Node child) { return child.parentHostname() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index bb06db3e78b..490fed681f9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -792,13 +792,13 @@ public class NodeRepository extends AbstractComponent { public boolean canAllocateTenantNodeTo(Node host) { if ( ! host.type().canRun(NodeType.tenant)) return false; + if (host.status().wantToRetire()) return false; + if (host.allocation().map(alloc -> alloc.membership().retired()).orElse(false)) return false; - // Do not allocate to hosts we want to retire or are currently retiring - if (host.status().wantToRetire() || host.allocation().map(alloc -> alloc.membership().retired()).orElse(false)) - return false; - - if ( ! zone.getCloud().dynamicProvisioning()) return host.state() == State.active; - else return EnumSet.of(State.active, State.ready, State.provisioned).contains(host.state()); + if ( zone.getCloud().dynamicProvisioning()) + return EnumSet.of(State.active, State.ready, State.provisioned).contains(host.state()); + else + return host.state() == State.active; } /** Returns the time keeper of this system */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index 672be25c5be..b508198db3a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -11,8 +11,8 @@ import java.util.Objects; /** * Capacity calculation for docker hosts. * <p> - * The calculations is based on an immutable copy of nodes that represents - * all capacities in the system - i.e. all nodes in the node repo give or take. + * The calculations are based on an immutable copy of nodes that represents + * all capacities in the system - i.e. all nodes in the node repo. * * @author smorgrav */ @@ -30,7 +30,7 @@ public class DockerHostCapacity { int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); if (result != 0) return result; - // If resources are equal we want to assign to the one with the most IPaddresses free + // If resources are equal we want to assign to the one with the most IP addresses free return freeIPs(hostB) - freeIPs(hostA); } @@ -65,9 +65,9 @@ public class DockerHostCapacity { NodeResources freeCapacityOf(Node host, boolean excludeInactive) { // Only hosts have free capacity - if (!host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0); - NodeResources hostResources = hostResourcesCalculator.advertisedResourcesOf(host.flavor()); + if ( ! host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0); + NodeResources hostResources = hostResourcesCalculator.advertisedResourcesOf(host.flavor()); return allNodes.childrenOf(host).asList().stream() .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) .map(node -> node.flavor().resources().justNumbers()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 30f9001093d..a1d8ffb03d3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -67,24 +67,22 @@ public class GroupPreparer { try (Mutex allocationLock = nodeRepository.lockUnallocated()) { // Create a prioritized set of nodes - LockedNodeList nodeList = nodeRepository.list(allocationLock); - NodePrioritizer prioritizer = new NodePrioritizer(nodeList, + LockedNodeList allNodes = nodeRepository.list(allocationLock); + NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, + highestIndex, nodeRepository); + + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, application, cluster, requestedNodes, spareCount, wantedGroups, - nodeRepository.nameResolver(), - nodeRepository.resourcesCalculator(), - allocateFully); - + allocateFully, + nodeRepository); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); prioritizer.addReadyNodes(); - prioritizer.addNewDockerNodes(nodeRepository::canAllocateTenantNodeTo); - // Allocate from the prioritized list - NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes, - highestIndex, nodeRepository); + prioritizer.addNewDockerNodes(); allocation.offer(prioritizer.prioritize()); if (dynamicProvisioningEnabled) { @@ -114,15 +112,15 @@ public class GroupPreparer { } if (! allocation.fulfilled() && requestedNodes.canFail()) - throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster + - " in " + application.toShortString() + + throw new OutOfCapacityException((cluster.group().isPresent() ? "Out of capacity on " + cluster.group().get() :"") + allocation.outOfCapacityDetails()); // Carry out and return allocation nodeRepository.reserve(allocation.reservableNodes()); nodeRepository.addDockerNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); - surplusActiveNodes.removeAll(allocation.surplusNodes()); - return allocation.finalNodes(surplusActiveNodes); + List<Node> acceptedNodes = allocation.finalNodes(); + surplusActiveNodes.removeAll(acceptedNodes); + return acceptedNodes; } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 83c68c91fc5..47d1b30a8e7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -316,10 +316,9 @@ class NodeAllocation { * Prefer to retire nodes of the wrong flavor. * Make as few changes to the retired set as possible. * - * @param surplusNodes this will add nodes not any longer needed by this group to this list * @return the final list of nodes */ - List<Node> finalNodes(List<Node> surplusNodes) { + List<Node> finalNodes() { int currentRetiredCount = (int) nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), currentRetiredCount) - currentRetiredCount; @@ -327,7 +326,6 @@ class NodeAllocation { for (PrioritizableNode node : byDecreasingIndex(nodes)) { if ( ! node.node.allocation().get().membership().retired() && node.node.state() == Node.State.active) { node.node = node.node.retire(Agent.application, nodeRepository.clock().instant()); - surplusNodes.add(node.node); // offer this node to other groups if (--deltaRetiredCount == 0) break; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index a7d83bbfad9..8a15c058ff4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -11,6 +11,7 @@ import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; @@ -42,7 +43,7 @@ public class NodePrioritizer { private final NodeSpec requestedNodes; private final ApplicationId application; private final ClusterSpec clusterSpec; - private final NameResolver nameResolver; + private final NodeRepository nodeRepository; private final boolean isDocker; private final boolean isAllocatingForReplacement; private final boolean isTopologyChange; @@ -52,16 +53,15 @@ public class NodePrioritizer { private final Set<Node> spareHosts; NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, - int spares, int wantedGroups, NameResolver nameResolver, HostResourcesCalculator hostResourcesCalculator, - boolean allocateFully) { + int spares, int wantedGroups, boolean allocateFully, NodeRepository nodeRepository) { this.allNodes = allNodes; - this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); + this.capacity = new DockerHostCapacity(allNodes, nodeRepository.resourcesCalculator()); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; this.application = application; - this.nameResolver = nameResolver; this.spareHosts = findSpareHosts(allNodes, capacity, spares); this.allocateFully = allocateFully; + this.nodeRepository = nodeRepository; NodeList nodesInCluster = allNodes.owner(application).type(clusterSpec.type()).cluster(clusterSpec.id()); NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired(); @@ -78,9 +78,8 @@ public class NodePrioritizer { .filter(clusterSpec.group()::equals) .count(); - this.isAllocatingForReplacement = isReplacement( - nodesInCluster.size(), - nodesInCluster.state(Node.State.failed).size()); + this.isAllocatingForReplacement = isReplacement(nodesInCluster.size(), + nodesInCluster.state(Node.State.failed).size()); this.isDocker = resources(requestedNodes) != null; } @@ -119,11 +118,11 @@ public class NodePrioritizer { } /** Add a node on each docker host with enough capacity for the requested flavor */ - void addNewDockerNodes(Predicate<Node> canAllocateTenantNodeTo) { + void addNewDockerNodes() { if ( ! isDocker) return; LockedNodeList candidates = allNodes - .filter(node -> node.type() != NodeType.host || canAllocateTenantNodeTo.test(node)) + .filter(node -> node.type() != NodeType.host || nodeRepository.canAllocateTenantNodeTo(node)) .filter(node -> node.reservedTo().isEmpty() || node.reservedTo().get().equals(application.tenant())); if (allocateFully) { @@ -142,25 +141,20 @@ public class NodePrioritizer { } private void addNewDockerNodesOn(LockedNodeList candidates) { - NodeResources wantedResources = resources(requestedNodes); - for (Node host : candidates) { - boolean hostHasCapacityForWantedFlavor = capacity.hasCapacity(host, wantedResources); - boolean conflictingCluster = allNodes.childrenOf(host).owner(application).asList().stream() - .anyMatch(child -> child.allocation().get().membership().cluster().id().equals(clusterSpec.id())); - - if (!hostHasCapacityForWantedFlavor || conflictingCluster) continue; + if ( ! capacity.hasCapacity(host, resources(requestedNodes))) continue; + if ( ! allNodes.childrenOf(host).owner(application).cluster(clusterSpec.id()).isEmpty()) continue; - log.log(Level.FINE, "Trying to add new Docker node on " + host); Optional<IP.Allocation> allocation; try { - allocation = host.ipConfig().pool().findAllocation(allNodes, nameResolver); + allocation = host.ipConfig().pool().findAllocation(allNodes, nodeRepository.nameResolver()); if (allocation.isEmpty()) continue; // No free addresses in this pool } catch (Exception e) { log.log(Level.WARNING, "Failed allocating IP address on " + host.hostname(), e); continue; } + log.log(Level.FINE, "Creating new docker node on " + host); Node newNode = Node.createDockerNode(allocation.get().addresses(), allocation.get().hostname(), host.hostname(), @@ -204,9 +198,8 @@ public class NodePrioritizer { * parameters to the priority sorting procedure. */ private PrioritizableNode toPrioritizable(Node node, boolean isSurplusNode, boolean isNewNode) { - PrioritizableNode.Builder builder = new PrioritizableNode.Builder(node) - .surplusNode(isSurplusNode) - .newNode(isNewNode); + PrioritizableNode.Builder builder = new PrioritizableNode.Builder(node).surplusNode(isSurplusNode) + .newNode(isNewNode); allNodes.parentOf(node).ifPresent(parent -> { NodeResources parentCapacity = capacity.freeCapacityOf(parent, false); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 913357b16ca..bd92357ea79 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -74,7 +74,6 @@ public class NodeRepositoryProvisioner implements Provisioner { this.preparer = new Preparer(nodeRepository, zone.environment() == Environment.prod ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, provisionServiceProvider.getHostProvisioner(), - provisionServiceProvider.getHostResourcesCalculator(), flagSource, loadBalancerProvisioner); this.activator = new Activator(nodeRepository, loadBalancerProvisioner); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index f88caffa6c6..4452fc21c52 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.Node; @@ -28,7 +29,7 @@ class Preparer { private final int spareCount; public Preparer(NodeRepository nodeRepository, int spareCount, Optional<HostProvisioner> hostProvisioner, - HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource, + FlagSource flagSource, Optional<LoadBalancerProvisioner> loadBalancerProvisioner) { this.nodeRepository = nodeRepository; this.spareCount = spareCount; @@ -38,9 +39,17 @@ class Preparer { /** Prepare all required resources for the given application and cluster */ public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) { - var nodes = prepareNodes(application, cluster, requestedNodes, wantedGroups); - prepareLoadBalancer(application, cluster, requestedNodes); - return nodes; + try { + var nodes = prepareNodes(application, cluster, requestedNodes, wantedGroups); + prepareLoadBalancer(application, cluster, requestedNodes); + return nodes; + } + catch (OutOfCapacityException e) { + throw new OutOfCapacityException("Could not satisfy " + requestedNodes + + ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") + + " in " + application + " " + cluster + + ": " + e.getMessage()); + } } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java index 8cfcbcb3797..a60dee6bb0c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java @@ -33,7 +33,7 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { /** True if the node is allocated to a host that should be dedicated as a spare */ final boolean violatesSpares; - /** True if this is a node that has been retired earlier in the allocation process */ + /** True if this node belongs to a group which will not be needed after this deployment */ final boolean isSurplusNode; /** This node does not exist in the node repository yet */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 441538fc305..e4d464840ea 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -249,7 +249,8 @@ public class DockerProvisioningTest { assertEquals("No room for 3 nodes as 2 of 4 hosts are exclusive", "Could not satisfy request for 3 nodes with " + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local] " + - "for container cluster 'myContainer' group 0 6.39 in tenant1.app1: " + + "in tenant1.app1 container cluster 'myContainer' 6.39: " + + "Out of capacity on group 0: " + "Not enough nodes available due to host exclusivity constraints, " + "insufficient nodes available on separate physical hosts", e.getMessage()); @@ -282,7 +283,7 @@ public class DockerProvisioningTest { try { ProvisioningTester tester = new ProvisioningTester.Builder() .zone(new Zone(Environment.prod, RegionName.from("us-east-1"))).build(); - ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application1 = tester.makeApplicationId("app1"); tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost1"); tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost2"); @@ -292,7 +293,11 @@ public class DockerProvisioningTest { dockerFlavor.with(NodeResources.StorageType.remote)); } catch (OutOfCapacityException e) { - assertTrue(e.getMessage().startsWith("Could not satisfy request for 2 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote]")); + assertEquals("Could not satisfy request for 2 nodes with " + + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote] " + + "in tenant.app1 content cluster 'myContent'" + + " 6.42: Out of capacity on group 0", + e.getMessage()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 050f3b7e865..3b03e4e9d91 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -40,9 +40,9 @@ public class MultigroupProvisioningTest { public void test_provisioning_of_multiple_groups() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application1 = tester.makeApplicationId("app1"); - tester.makeReadyNodes(21, small); + tester.makeReadyNodes(31, small); deploy(application1, 6, 1, small, tester); deploy(application1, 6, 2, small, tester); @@ -86,10 +86,10 @@ public class MultigroupProvisioningTest { public void test_provisioning_of_multiple_groups_after_flavor_migration() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application1 = tester.makeApplicationId("app1"); tester.makeReadyNodes(10, small); - tester.makeReadyNodes(10, large); + tester.makeReadyNodes(16, large); deploy(application1, 8, 1, small, tester); deploy(application1, 8, 1, large, tester); @@ -125,10 +125,10 @@ public class MultigroupProvisioningTest { public void test_provisioning_of_multiple_groups_after_flavor_migration_and_exiration() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application1 = tester.makeApplicationId("app1"); tester.makeReadyNodes(10, small); - tester.makeReadyNodes(10, large); + tester.makeReadyNodes(16, large); deploy(application1, 8, 1, small, tester); deploy(application1, 8, 1, large, tester); @@ -164,13 +164,8 @@ public class MultigroupProvisioningTest { int nodeCount = capacity.minResources().nodes(); NodeResources nodeResources = capacity.minResources().nodeResources(); - int previousActiveNodeCount = tester.getNodes(application, Node.State.active).resources(nodeResources).size(); - tester.activate(application, prepare(application, capacity, tester)); - assertEquals("Superfluous nodes are retired, but no others - went from " + previousActiveNodeCount + " to " + nodeCount + " nodes", - Math.max(0, previousActiveNodeCount - capacity.minResources().nodes()), - tester.getNodes(application, Node.State.active).retired().resources(nodeResources).size()); - assertEquals("Other flavors are retired", + assertEquals("Nodes of wrong size are retired", 0, tester.getNodes(application, Node.State.active).not().retired().not().resources(nodeResources).size()); // Check invariants for all nodes diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 03c07515cd5..44688c61b34 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -486,6 +486,40 @@ public class ProvisioningTest { app1, cluster1); } + /** + * When increasing the number of groups without changing node count, we need to provison new nodes for + * the new groups since although we can remove nodes from existing groups without losing data we + * cannot do so without losing coverage. + */ + @Test + public void test_change_group_layout() { + Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .flavors(List.of(hostFlavor)) + .build(); + tester.makeReadyHosts(6, hostFlavor.resources()).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + // Deploy with 1 group + System.out.println("--- Deploying 1 group"); + tester.activate(app1, cluster1, Capacity.from(resources(4, 1, 10, 30, 10))); + assertEquals(4, tester.getNodes(app1, Node.State.active).size()); + assertEquals(4, tester.getNodes(app1, Node.State.active).group(0).size()); + assertEquals(0, tester.getNodes(app1, Node.State.active).group(0).retired().size()); + + // Split into 2 groups + System.out.println("--- Deploying 2 groups"); + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 10, 30, 10))); + assertEquals(6, tester.getNodes(app1, Node.State.active).size()); + assertEquals(4, tester.getNodes(app1, Node.State.active).group(0).size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(0).retired().size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(1).size()); + assertEquals(0, tester.getNodes(app1, Node.State.active).group(1).retired().size()); + } + + @Test(expected = IllegalArgumentException.class) public void prod_deployment_requires_redundancy() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); diff --git a/parent/pom.xml b/parent/pom.xml index 2faabde5241..5533e37d898 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -220,6 +220,7 @@ <configuration> <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> <systemPropertyVariables> + <jdisc.logger.level>INFO</jdisc.logger.level> <java.io.tmpdir>${project.build.directory}</java.io.tmpdir> </systemPropertyVariables> <trimStackTrace>false</trimStackTrace> diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index c7000119861..719ba359ccf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -212,6 +212,7 @@ Proton::Proton(const config::ConfigUri & configUri, _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), _warmupExecutor(), _sharedExecutor(), + _compile_cache_executor_binding(), _queryLimiter(), _clock(0.001), _threadPool(128 * 1024), @@ -302,6 +303,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) const size_t sharedThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); _sharedExecutor = std::make_unique<vespalib::BlockingThreadStackExecutor>(sharedThreads, 128*1024, sharedThreads*16, proton_shared_executor); + _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(*_sharedExecutor); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128 * 1024, initialize_executor); @@ -454,6 +456,7 @@ Proton::~Proton() _persistenceEngine.reset(); _tls.reset(); _warmupExecutor.reset(); + _compile_cache_executor_binding.reset(); _sharedExecutor.reset(); _clock.stop(); LOG(debug, "Explicit destructor done"); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 4c9d4c77cc4..d0b76bd5804 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -26,6 +26,7 @@ #include <vespa/vespalib/net/json_handler_repo.h> #include <vespa/vespalib/net/state_explorer.h> #include <vespa/vespalib/util/varholder.h> +#include <vespa/eval/eval/llvm/compile_cache.h> #include <mutex> #include <shared_mutex> @@ -112,6 +113,7 @@ private: ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor; std::unique_ptr<vespalib::ThreadStackExecutorBase> _sharedExecutor; + vespalib::eval::CompileCache::ExecutorBinding::UP _compile_cache_executor_binding; matching::QueryLimiter _queryLimiter; vespalib::Clock _clock; FastOS_ThreadPool _threadPool; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 9ef038b7325..e169f51ef9d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -9,6 +9,7 @@ #include <vespa/searchcore/proton/common/indexschema_inspector.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> +#include <vespa/eval/eval/llvm/compile_cache.h> using namespace vespa::config::search; using namespace config; @@ -87,6 +88,9 @@ void SearchableDocSubDBConfigurer::reconfigureSearchView(MatchView::SP matchView) { SearchView::SP curr = _searchView.get(); + // make sure the initial search does not spend time waiting for + // expression compilation completion during rank program setup. + vespalib::eval::CompileCache::wait_pending(); _searchView.set(SearchView::create(curr->getSummarySetup(), std::move(matchView))); } diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h index 12535ac105c..b40b27596dc 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h @@ -121,15 +121,6 @@ public: int fieldIndexToSemanticDistance(int j, uint32_t zeroJ) const; /** - * Returns the query environment of this. This contains information about the query. - * - * @return The query environment. - */ - const fef::IQueryEnvironment &getQueryEnvironment() const { - return _splitter; - } - - /** * Returns the id of the searched field. * * @return The field id. diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp index 6a56cb606d2..dcdaf1681a4 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp @@ -8,9 +8,7 @@ #include <cmath> #include <cstdlib> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { Metrics::Metrics(const Computer *source) : _source(source), @@ -337,7 +335,4 @@ Metrics::toString() const return vespalib::make_string("Metrics(match %f)", getMatch()); } - -} // fieldmatch -} // features -} // search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h index eadda3209ad..a5f0934cd64 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h @@ -5,9 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vector> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { class Computer; @@ -557,7 +555,4 @@ private: uint32_t _queryLength; // num terms searching this field }; -} // fieldmatch -} // features -} // search - +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp index facf1a5f64a..f17d3cffcd2 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp @@ -4,9 +4,7 @@ #include <vespa/log/log.h> LOG_SETUP(".features.fieldmatch.params"); -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { Params::Params() : _proximityLimit(10), @@ -40,5 +38,3 @@ Params::valid() } } -} -} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/params.h b/searchlib/src/vespa/searchlib/features/fieldmatch/params.h index 21c35f19472..dcbd61676ad 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/params.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/params.h @@ -5,9 +5,7 @@ #include <vector> #include <cstdint> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { /** * The parameters to a string match metric calculator. @@ -256,7 +254,4 @@ private: std::vector<feature_t> _proximityTable; }; -} // fieldmatch -} // features -} // search - +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp index 8f5f86d1628..82f2beeea32 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp @@ -5,9 +5,7 @@ #include "metrics.h" #include <vespa/vespalib/util/stringfmt.h> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { SegmentStart::SegmentStart(Computer *owner, const Metrics & metrics, uint32_t previousJ, uint32_t i, uint32_t j) : _owner(owner), @@ -95,6 +93,4 @@ SegmentStart::toString() { } } -} // fieldmatch -} // features -} // search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.h b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.h index dd4b9406036..9b3fbc7b987 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.h @@ -5,9 +5,7 @@ #include <limits> #include "metrics.h" -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { /** * <p>Information on segment start points stored temporarily during string match metric calculation.</p> @@ -179,7 +177,4 @@ private: bool _open; // There are possibly more j's to try at this starting point. }; -} // fieldmatch -} // features -} // search - +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp index 045bc277454..ca3b79f17bc 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp @@ -3,9 +3,7 @@ #include "simplemetrics.h" #include <vespa/vespalib/stllike/asciistream.h> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { SimpleMetrics::SimpleMetrics(const Params & params) : _params(params), @@ -30,7 +28,4 @@ vespalib::string SimpleMetrics::toString() const return ss.str(); } - -} // fieldmatch -} // features -} // search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.h b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.h index 6c5a67884d8..ee1c11215d1 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.h @@ -5,9 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include "params.h" -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { /** * The collection of simple metrics calculated when traversing the query terms of the query environment. @@ -180,7 +178,4 @@ public: vespalib::string toString() const; }; -} // fieldmatch -} // features -} // search - +} diff --git a/searchlib/src/vespa/searchlib/fef/phrasesplitter.h b/searchlib/src/vespa/searchlib/fef/phrasesplitter.h index 25944158445..8b399885496 100644 --- a/searchlib/src/vespa/searchlib/fef/phrasesplitter.h +++ b/searchlib/src/vespa/searchlib/fef/phrasesplitter.h @@ -55,7 +55,6 @@ private: TermFieldHandle _skipHandles; // how many handles to skip void considerTerm(uint32_t termIdx, const ITermData &term, std::vector<PhraseTerm> &phraseTerms, uint32_t fieldId); - void splitPhrase(const ITermData &phrase, std::vector<PhraseTerm> &phraseTerms, uint32_t fieldId); TermFieldMatchData *resolveSplittedTermField(TermFieldHandle handle) { return &_termMatches[handle - _skipHandles]; diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 16e6d402764..7a82187f355 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -262,9 +262,9 @@ TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_corr { set_field("smap", false); assertWritten("[ { key: \"k1.1\", value: { fval: 110.0, name: \"n1.1\", val: 10} }, { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); - assertWritten("[ { key: \"k2\", value: { fval: 120.0, name: \"n2\", val: 20} }, { value: { fval: 121.0, val: 21 } }]", 2); + assertWritten("[ { key: \"k2\", value: { fval: 120.0, name: \"n2\", val: 20} }, { key: \"\", value: { fval: 121.0, val: 21 } }]", 2); assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} }, { key: \"k3.2\", value: { fval: 131.0, name: \"n3.2\"} } ]", 3); - assertWritten("[ { value: { } }, { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); + assertWritten("[ { key: \"\", value: { } }, { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); assertWritten("null", 5); } @@ -272,9 +272,9 @@ TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_corr { set_field("map", false); assertWritten("[ { key: \"k1.1\", value: \"n1.1\" }, { key: \"k1.2\", value: \"n1.2\"}]", 1); - assertWritten("[ { key: \"k2\"}]", 2); - assertWritten("[ { key: \"k3.1\", value: \"n3.1\" }, { value: \"n3.2\"} ]", 3); - assertWritten("[ { }, { key: \"k4.2\", value: \"n4.2\" } ]", 4); + assertWritten("[ { key: \"k2\", value: \"\" }]", 2); + assertWritten("[ { key: \"k3.1\", value: \"n3.1\" }, { key: \"\", value: \"n3.2\"} ]", 3); + assertWritten("[ { key: \"\", value: \"\" }, { key: \"k4.2\", value: \"n4.2\" } ]", 4); assertWritten("null", 5); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp index a1f3ce4d392..4e3cbaa512d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -30,7 +30,8 @@ public: const std::vector<vespalib::string> &attributeNames, IAttributeContext &context, const vespalib::string &field_name, - const MatchingElements* matching_elements); + const MatchingElements* matching_elements, + bool is_map_of_scalar); ~ArrayAttributeFieldWriterState() override; void insert_element(uint32_t element_index, Cursor &array); void insertField(uint32_t docId, vespalib::slime::Inserter &target) override; @@ -40,10 +41,11 @@ ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector const std::vector<vespalib::string> &attributeNames, IAttributeContext &context, const vespalib::string &field_name, - const MatchingElements *matching_elements) + const MatchingElements *matching_elements, + bool is_map_of_scalar) : DocsumFieldWriterState(), _writers(), - _field_name(field_name), + _field_name(field_name), _matching_elements(matching_elements) { size_t fields = fieldNames.size(); @@ -51,7 +53,7 @@ ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector for (uint32_t field = 0; field < fields; ++field) { const IAttributeVector *attr = context.getAttribute(attributeNames[field]); if (attr != nullptr) { - _writers.emplace_back(AttributeFieldWriter::create(fieldNames[field], *attr)); + _writers.emplace_back(AttributeFieldWriter::create(fieldNames[field], *attr, is_map_of_scalar)); } } } @@ -106,7 +108,8 @@ ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fie std::shared_ptr<StructFieldMapper> struct_field_mapper) : AttributeCombinerDFW(fieldName, filter_elements, std::move(struct_field_mapper)), _fields(fields_resolver.get_array_fields()), - _attributeNames(fields_resolver.get_array_attributes()) + _attributeNames(fields_resolver.get_array_attributes()), + _is_map_of_scalar(fields_resolver.is_map_of_scalar()) { if (filter_elements && _struct_field_mapper && !_struct_field_mapper->is_struct_field(fieldName)) { fields_resolver.apply_to(*_struct_field_mapper); @@ -118,7 +121,8 @@ ArrayAttributeCombinerDFW::~ArrayAttributeCombinerDFW() = default; std::unique_ptr<DocsumFieldWriterState> ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context, const MatchingElements* matching_elements) { - return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context, _fieldName, matching_elements); + return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context, + _fieldName, matching_elements, _is_map_of_scalar); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h index c3e686965cf..2fda1f69572 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -20,6 +20,7 @@ class ArrayAttributeCombinerDFW : public AttributeCombinerDFW { std::vector<vespalib::string> _fields; std::vector<vespalib::string> _attributeNames; + bool _is_map_of_scalar; std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) override; public: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp index 99c5d8b7f30..91fec74fce9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp @@ -46,6 +46,15 @@ public: void print(uint32_t idx, Cursor &cursor) override; }; +class WriteStringFieldNeverSkip : public WriteField<search::attribute::ConstCharContent> +{ +public: + WriteStringFieldNeverSkip(vespalib::Memory fieldName, + const IAttributeVector &attr) + : WriteField(fieldName, attr) {} + ~WriteStringFieldNeverSkip() override {} + void print(uint32_t idx, Cursor &cursor) override; +}; class WriteFloatField : public WriteField<search::attribute::FloatContent> { @@ -104,6 +113,17 @@ WriteStringField::print(uint32_t idx, Cursor &cursor) } } +void +WriteStringFieldNeverSkip::print(uint32_t idx, Cursor &cursor) +{ + if (idx < _size) { + const char *s = _content[idx]; + cursor.setString(_fieldName, vespalib::Memory(s)); + } else { + cursor.setString(_fieldName, vespalib::Memory("")); + } +} + WriteFloatField::WriteFloatField(vespalib::Memory fieldName, const IAttributeVector &attr) : WriteField(fieldName, attr) @@ -147,7 +167,7 @@ WriteIntField::print(uint32_t idx, Cursor &cursor) } std::unique_ptr<AttributeFieldWriter> -AttributeFieldWriter::create(vespalib::Memory fieldName, const IAttributeVector &attr) +AttributeFieldWriter::create(vespalib::Memory fieldName, const IAttributeVector &attr, bool keep_empty_strings) { switch (attr.getBasicType()) { case BasicType::INT8: @@ -162,7 +182,11 @@ AttributeFieldWriter::create(vespalib::Memory fieldName, const IAttributeVector case BasicType::DOUBLE: return std::make_unique<WriteFloatField>(fieldName, attr); case BasicType::STRING: - return std::make_unique<WriteStringField>(fieldName, attr); + if (keep_empty_strings) { + return std::make_unique<WriteStringFieldNeverSkip>(fieldName, attr); + } else { + return std::make_unique<WriteStringField>(fieldName, attr); + } default: assert(false); abort(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h index 3fed6edb1b5..b6c6bc82325 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h @@ -27,7 +27,7 @@ public: virtual ~AttributeFieldWriter(); virtual void fetch(uint32_t docId) = 0; virtual void print(uint32_t idx, vespalib::slime::Cursor &cursor) = 0; - static std::unique_ptr<AttributeFieldWriter> create(vespalib::Memory fieldName, const search::attribute::IAttributeVector &attr); + static std::unique_ptr<AttributeFieldWriter> create(vespalib::Memory fieldName, const search::attribute::IAttributeVector &attr, bool keep_empty_strings = false); uint32_t size() const { return _size; } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp index c29f0324ce2..749fc2941a6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp @@ -22,12 +22,14 @@ StructFieldsResolver::StructFieldsResolver(const vespalib::string& field_name, c _array_fields(), _array_attributes(), _has_map_key(false), + _has_map_value(false), _error(false) { std::vector<const search::attribute::IAttributeVector *> attrs; attr_ctx.getAttributeList(attrs); vespalib::string prefix = field_name + "."; _map_key_attribute = prefix + "key"; + vespalib::string map_value_attribute_name = prefix + "value"; vespalib::string value_prefix = prefix + "value."; for (const auto attr : attrs) { vespalib::string name = attr->getName(); @@ -45,6 +47,8 @@ StructFieldsResolver::StructFieldsResolver(const vespalib::string& field_name, c _array_fields.emplace_back(name.substr(prefix.size())); if (name == _map_key_attribute) { _has_map_key = true; + } else if (name == map_value_attribute_name) { + _has_map_value = true; } } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h index 66d9fd69db4..b88e51a346b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h @@ -26,12 +26,18 @@ private: StringVector _array_fields; StringVector _array_attributes; bool _has_map_key; + bool _has_map_value; bool _error; public: StructFieldsResolver(const vespalib::string& field_name, const search::attribute::IAttributeContext& attr_ctx, bool require_all_struct_fields_as_attributes); ~StructFieldsResolver(); + bool is_map_of_scalar() const { return (_has_map_key && + _has_map_value && + (_array_fields.size() == 2u) && + _map_value_fields.empty()); + } bool is_map_of_struct() const { return !_map_value_fields.empty(); } const vespalib::string& get_map_key_attribute() const { return _map_key_attribute; } const StringVector& get_map_value_fields() const { return _map_value_fields; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp index d5922ddf46b..0a2c4726577 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp @@ -55,7 +55,7 @@ StructMapAttributeFieldWriterState::StructMapAttributeFieldWriterState(const ves { const IAttributeVector *attr = context.getAttribute(keyAttributeName); if (attr != nullptr) { - _keyWriter = AttributeFieldWriter::create(keyName, *attr); + _keyWriter = AttributeFieldWriter::create(keyName, *attr, true); } size_t fields = valueFieldNames.size(); _valueWriters.reserve(fields); diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index 220d0cbd6bb..53e7311fefc 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -15,7 +15,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * Tests that verify the health of production deployments of Vespa applications. * * Test classes annotated with this annotation are run during declared production tests. - * See <a href="https://cloud.vespa.ai/automated-deployments.html#production-tests">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java index 6853ebfc008..bef3eabcef6 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java @@ -15,7 +15,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * * Test classes annotated with this annotation are run in the first phase of automated staging tests, * to make the initial deployment similar to a production one. - * See <a href="https://cloud.vespa.ai/automated-deployments.html#staging-tests">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 4ecc3ce5722..59360b2753c 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java @@ -16,7 +16,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * * Test classes annotated with this annotation are run in the second phase of automated staging tests, * to verify the upgraded deployment. - * See <a href="https://cloud.vespa.ai/automated-deployments.html#staging-tests">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java index 4712c1b41ef..f01f2ca6c90 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java @@ -17,7 +17,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * Tests that compare the behaviour of a Vespa application deployment against a fixed specification. * * Test classes annotated with this annotation are run against a fresh deployment during automated system tests. - * See <a href="https://cloud.vespa.ai/automated-deployments.html#system-tests">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java index f1c5fbb0ba5..bdcfac2529e 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java @@ -13,7 +13,7 @@ import static java.util.Map.copyOf; /** * Metrics from a Vespa application {@link Endpoint}, indexed by their names, and optionally by a set of custom dimensions. * - * Metrics are collected from the <a href="https://docs.vespa.ai/documentation/reference/metrics-health-format.html">metrics</a> + * Metrics are collected from the <a href="https://docs.vespa.ai/documentation/reference/metrics.html">metrics</a> * API of a Vespa endpoint, and contain the current health status of the endpoint, values for all configured metrics in * that endpoint, and the time interval from which these metrics were sampled. * diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index 10995d3dd2d..c78be98e11e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -20,8 +20,8 @@ import java.util.Iterator; /** * Writes tensors on the JSON format used in Vespa tensor document fields: * A JSON map containing a 'cells' or 'values' array. - * See <a href="http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor"> - * http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor</a> + * See <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html"> + * https://docs.vespa.ai/documentation/reference/document-json-format.html</a> * * @author bratseth */ |