diff options
36 files changed, 401 insertions, 346 deletions
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 6a41ce66456..7f4273c827b 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -331,6 +331,8 @@ "methods": [ "public void <init>(java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.lang.String)", "public void <init>(java.util.Optional, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, java.util.Optional, java.util.List, java.util.List, java.lang.String, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List)", + "public java.util.Optional globalServiceId()", + "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", "public java.util.Optional majorVersion()", "public boolean canUpgradeAt(java.time.Instant)", "public boolean canChangeRevisionAt(java.time.Instant)", diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 7b4e91bfe4a..446dc8d1fc3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; -import com.yahoo.collections.Comparables; import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; @@ -73,7 +72,6 @@ public class DeploymentSpec { this.athenzService = athenzService; this.xmlForm = xmlForm; validateTotalDelay(steps); - validateUpgradePoliciesOfIncreasingConservativeness(steps); } // TODO: Remove after October 2019 @@ -149,23 +147,6 @@ public class DeploymentSpec { " but max 24 hours is allowed"); } - /** Throws an IllegalArgumentException if any instance has a looser upgrade policy than the previous */ - private void validateUpgradePoliciesOfIncreasingConservativeness(List<Step> steps) { - UpgradePolicy previous = Collections.min(List.of(UpgradePolicy.values())); - for (Step step : steps) { - UpgradePolicy strictest = previous; - List<DeploymentInstanceSpec> specs = instances(List.of(step)); - for (DeploymentInstanceSpec spec : specs) { - if (spec.upgradePolicy().compareTo(previous) < 0) - throw new IllegalArgumentException("Instance '" + spec.name() + "' cannot have a looser upgrade " + - "policy than the previous of '" + previous + "'"); - - strictest = Comparables.max(strictest, spec.upgradePolicy()); - } - previous = strictest; - } - } - // TODO: Remove after October 2019 private DeploymentInstanceSpec singleInstance() { return singleInstance(steps); @@ -180,6 +161,12 @@ public class DeploymentSpec { instances.stream().map(Step::toString).collect(Collectors.joining(","))); } + // TODO: Remove after October 2019 + public Optional<String> globalServiceId() { return singleInstance().globalServiceId(); } + + // TODO: Remove after October 2019 + public UpgradePolicy upgradePolicy() { return singleInstance().upgradePolicy(); } + /** Returns the major version this application is pinned to, or empty (default) to allow all major versions */ public Optional<Integer> majorVersion() { return majorVersion; } @@ -283,7 +270,7 @@ public class DeploymentSpec { private static List<DeploymentInstanceSpec> instances(List<DeploymentSpec.Step> steps) { return steps.stream() - .flatMap(step -> step instanceof ParallelZones ? ((ParallelZones) step).steps.stream() : List.of(step).stream()) + .flatMap(step -> step instanceof ParallelZones ? ((ParallelZones)step).steps.stream() : List.of(step).stream()) .filter(step -> step instanceof DeploymentInstanceSpec).map(DeploymentInstanceSpec.class::cast) .collect(Collectors.toList()); } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecDeprecatedAPITest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecDeprecatedAPITest.java index 8167ba7af11..5a8358e65c3 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecDeprecatedAPITest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecDeprecatedAPITest.java @@ -45,6 +45,7 @@ public class DeploymentSpecDeprecatedAPITest { assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); assertFalse(spec.includes(Environment.staging, Optional.empty())); assertFalse(spec.includes(Environment.prod, Optional.empty())); + assertFalse(spec.globalServiceId().isPresent()); } @Test @@ -77,6 +78,7 @@ public class DeploymentSpecDeprecatedAPITest { assertFalse(spec.includes(Environment.test, Optional.of(RegionName.from("region1")))); assertTrue(spec.includes(Environment.staging, Optional.empty())); assertFalse(spec.includes(Environment.prod, Optional.empty())); + assertFalse(spec.globalServiceId().isPresent()); } @Test @@ -109,6 +111,9 @@ public class DeploymentSpecDeprecatedAPITest { assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); assertFalse(spec.includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(spec.globalServiceId().isPresent()); + + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.upgradePolicy()); } @Test @@ -203,6 +208,7 @@ public class DeploymentSpecDeprecatedAPITest { assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-east1")))); assertTrue(spec.includes(Environment.prod, Optional.of(RegionName.from("us-west1")))); assertFalse(spec.includes(Environment.prod, Optional.of(RegionName.from("no-such-region")))); + assertFalse(spec.globalServiceId().isPresent()); } @Test @@ -217,6 +223,7 @@ public class DeploymentSpecDeprecatedAPITest { ); DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(spec.globalServiceId(), Optional.of("query")); } @Test(expected=IllegalArgumentException.class) @@ -240,6 +247,41 @@ public class DeploymentSpecDeprecatedAPITest { } @Test + public void productionSpecWithGlobalServiceIdBeforeStaging() { + StringReader r = new StringReader( + "<deployment>" + + " <test/>" + + " <prod global-service-id='qrs'>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + " <staging/>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("qrs", spec.globalServiceId().get()); + } + + @Test + public void productionSpecWithUpgradePolicy() { + StringReader r = new StringReader( + "<deployment>" + + " <upgrade policy='canary'/>" + + " <prod>" + + " <region active='true'>us-west-1</region>" + + " <region active='true'>us-central-1</region>" + + " <region active='true'>us-east-3</region>" + + " </prod>" + + "</deployment>" + ); + + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals("canary", spec.upgradePolicy().toString()); + } + + @Test public void maxDelayExceeded() { try { StringReader r = new StringReader( @@ -265,6 +307,8 @@ public class DeploymentSpecDeprecatedAPITest { @Test public void testEmpty() { + assertFalse(DeploymentSpec.empty.globalServiceId().isPresent()); + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); assertTrue(DeploymentSpec.empty.steps().isEmpty()); assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); } @@ -322,7 +366,7 @@ public class DeploymentSpecDeprecatedAPITest { " <block-change days='mon,tue' hours='15-16'/>\n" + "</deployment>" ); - DeploymentSpec.fromXml(r); + DeploymentSpec spec = DeploymentSpec.fromXml(r); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 080982cb785..7b571417ef8 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -310,16 +310,16 @@ public class DeploymentSpecTest { "<deployment version='1.0'>" + " <upgrade policy='canary'/>" + " <instance id='instance1'>" + + " <upgrade policy='conservative'/>" + " </instance>" + " <instance id='instance2'>" + - " <upgrade policy='conservative'/>" + " </instance>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals("canary", spec.requireInstance("instance1").upgradePolicy().toString()); - assertEquals("conservative", spec.requireInstance("instance2").upgradePolicy().toString()); + assertEquals("conservative", spec.requireInstance("instance1").upgradePolicy().toString()); + assertEquals("canary", spec.requireInstance("instance2").upgradePolicy().toString()); } @Test @@ -351,6 +351,7 @@ public class DeploymentSpecTest { @Test public void testEmpty() { assertFalse(DeploymentSpec.empty.requireInstance("default").globalServiceId().isPresent()); + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); assertTrue(DeploymentSpec.empty.steps().isEmpty()); assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); } @@ -501,72 +502,6 @@ public class DeploymentSpecTest { } @Test(expected = IllegalArgumentException.class) - public void deploymentSpecWithIncreasinglyStrictUpgradePolicies() { - StringReader r = new StringReader( - "<deployment version='1.0'>" + - " <instance id='instance1'>" + - " <upgrade policy='conservative'/>" + - " </instance>" + - " <instance id='instance2' />" + - "</deployment>" - ); - DeploymentSpec.fromXml(r); - } - - @Test(expected = IllegalArgumentException.class) - public void deploymentSpecWithIncreasinglyStrictUpgradePoliciesInParallel() { - StringReader r = new StringReader( - "<deployment version='1.0'>" + - " <instance />" + - " <parallel>" + - " <instance id='instance1'>" + - " <upgrade policy='conservative'/>" + - " </instance>" + - " <instance id='instance2'>" + - " <upgrade policy='canary'/>" + - " </instance>" + - " </parallel>" + - "</deployment>" - ); - DeploymentSpec.fromXml(r); - } - - @Test(expected = IllegalArgumentException.class) - public void deploymentSpecWithIncreasinglyStrictUpgradePoliciesAfterParallel() { - StringReader r = new StringReader( - "<deployment version='1.0'>" + - " <parallel>" + - " <instance id='instance1'>" + - " <upgrade policy='conservative'/>" + - " </instance>" + - " <instance id='instance2'>" + - " <upgrade policy='canary'/>" + - " </instance>" + - " </parallel>" + - " <instance />" + - "</deployment>" - ); - DeploymentSpec.fromXml(r); - } - - @Test - public void deploymentSpecWithDifferentUpgradePoliciesInParallel() { - StringReader r = new StringReader( - "<deployment version='1.0'>" + - " <parallel>" + - " <instance id='instance1'>" + - " <upgrade policy='conservative'/>" + - " </instance>" + - " <instance id='instance2' />" + - " </parallel>" + - "</deployment>" - ); - DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(DeploymentSpec.UpgradePolicy.conservative, spec.requireInstance("instance1").upgradePolicy()); - assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, spec.requireInstance("instance2").upgradePolicy()); - } - - @Test(expected = IllegalArgumentException.class) public void deploymentSpecWithIllegallyOrderedDeploymentSpec1() { StringReader r = new StringReader( "<deployment>" + diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java index 25c37a2a240..7805b73cc6a 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java @@ -253,6 +253,7 @@ public class DeploymentSpecWithoutInstanceTest { @Test public void testEmpty() { assertFalse(DeploymentSpec.empty.requireInstance("default").globalServiceId().isPresent()); + assertEquals(DeploymentSpec.UpgradePolicy.defaultPolicy, DeploymentSpec.empty.upgradePolicy()); assertTrue(DeploymentSpec.empty.steps().isEmpty()); assertEquals("<deployment version='1.0'/>", DeploymentSpec.empty.xmlForm()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java index 8c0c6a74037..2fab941b10e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java @@ -7,11 +7,14 @@ import com.yahoo.searchdefinition.derived.SummaryClass; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Optional; import java.util.logging.Level; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; + /** * Emits a warning for summaries which accesses disk. * @@ -41,7 +44,7 @@ public class SummaryDiskAccessValidator extends Processor { if (field == null && ! source.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) throw new IllegalArgumentException(summaryField + " in " + summary + " references " + source + ", but this field does not exist"); - if ( ! isInMemory(field) && ! summary.isFromDisk()) { + if ( ! isInMemory(field, summaryField) && ! summary.isFromDisk()) { deployLogger.log(Level.WARNING, summaryField + " in " + summary + " references " + source + ", which is not an attribute: Using this " + "summary will cause disk accesses. " + @@ -52,8 +55,13 @@ public class SummaryDiskAccessValidator extends Processor { } } - private boolean isInMemory(ImmutableSDField field) { + private boolean isInMemory(ImmutableSDField field, SummaryField summaryField) { if (field == null) return false; // For DOCUMENT_ID_FIELD, which may be implicit, but is then not in memory + if (isComplexFieldWithOnlyStructFieldAttributes(field) && + (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER || + summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER)) { + return true; + } return field.doesAttributing(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index 4d02082217c..e05b2d27e09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -7,23 +7,18 @@ import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.cluster.QrMonitorConfig; -import com.yahoo.search.config.dispatchprototype.SearchNodesConfig; -import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.searchchain.model.federation.FederationOptions; import com.yahoo.search.searchchain.model.federation.LocalProviderSpec; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.IndexedSearchCluster; -import com.yahoo.vespa.model.search.SearchNode; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; - /** * Config producer for search chain responsible for sending queries to a local cluster. * @@ -34,8 +29,7 @@ public class LocalProvider extends Provider implements ClusterConfig.Producer, AttributesConfig.Producer, QrMonitorConfig.Producer, - RankProfilesConfig.Producer, - SearchNodesConfig.Producer { + RankProfilesConfig.Producer { private final LocalProviderSpec providerSpec; private volatile AbstractSearchCluster searchCluster; @@ -72,22 +66,6 @@ public class LocalProvider extends Provider implements } } - @Override - public void getConfig(final SearchNodesConfig.Builder builder) { - if (!(searchCluster instanceof IndexedSearchCluster)) { - log.warning("Could not build SearchNodesConfig: Only supported for IndexedSearchCluster, got " - + searchCluster.getClass().getCanonicalName()); - return; - } - final IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster) searchCluster; - for (final SearchNode searchNode : indexedSearchCluster.getSearchNodes()) { - builder.search_node( - new SearchNodesConfig.Search_node.Builder() - .host(searchNode.getHostName()) - .port(searchNode.getDispatchPort())); - } - } - private void addProviderSearchers() { for (ChainedComponentModel searcherModel : LocalProviderSpec.searcherModels) { addInnerComponent(new Searcher<>(searcherModel)); @@ -124,7 +102,7 @@ public class LocalProvider extends Provider implements return providerSpec.clusterName; } - public void setSearchCluster(AbstractSearchCluster searchCluster) { + void setSearchCluster(AbstractSearchCluster searchCluster) { this.searchCluster = searchCluster; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 3ea36dc800e..64ac00020f7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -302,7 +302,6 @@ public class IndexedSearchCluster extends SearchCluster nodeBuilder.group(node.getNodeSpec().groupIndex()); nodeBuilder.host(node.getHostName()); nodeBuilder.port(node.getRpcPort()); - nodeBuilder.fs4port(node.getDispatchPort()); builder.node(nodeBuilder); } if (useAdaptiveDispatch) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java index c7d69b6818a..7a3fc7dd715 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java @@ -10,7 +10,6 @@ package com.yahoo.vespa.model.search; public interface SearchInterface { NodeSpec getNodeSpec(); - String getDispatcherConnectSpec(); String getHostName(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index fe1a61086f2..a174e7c38e8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -64,8 +64,8 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private static final int RPC_PORT = 0; - private static final int FS4_PORT = 1; - private static final int FUTURE_HEALTH_PORT = 2; + private static final int UNUSED_1 = 1; + private static final int UNUSED_2 = 2; private static final int UNUSED_3 = 3; private static final int HEALTH_PORT = 4; @@ -123,8 +123,8 @@ public class SearchNode extends AbstractService implements this.clusterName = clusterName; this.flushOnShutdown = flushOnShutdown; portsMeta.on(RPC_PORT).tag("rpc").tag("rtc").tag("admin").tag("status"); - portsMeta.on(FS4_PORT).tag("fs4"); - portsMeta.on(FUTURE_HEALTH_PORT).tag("unused"); + portsMeta.on(UNUSED_1).tag("unused"); + portsMeta.on(UNUSED_2).tag("unused"); portsMeta.on(UNUSED_3).tag("unused"); portsMeta.on(HEALTH_PORT).tag("http").tag("json").tag("health").tag("state"); // Properties are set in DomSearchBuilder @@ -156,7 +156,7 @@ public class SearchNode extends AbstractService implements this.redundancy = redundancy; } - public void updatePartition(int partitionId) { + void updatePartition(int partitionId) { nodeSpec = new NodeSpec(nodeSpec.groupIndex(), partitionId); } @@ -169,8 +169,8 @@ public class SearchNode extends AbstractService implements public void allocatePorts(int start, PortAllocBridge from) { // NB: ignore "start" from.allocatePort("rpc"); - from.allocatePort("fs4"); - from.allocatePort("future/4"); + from.allocatePort("unused/1"); + from.allocatePort("unused/2"); from.allocatePort("unused/3"); from.allocatePort("health"); } @@ -208,20 +208,6 @@ public class SearchNode extends AbstractService implements return distributionKey; } - /** - * Returns the connection spec string that resolves to the dispatcher service - * on this node. - * - * @return The connection string. - */ - public String getDispatcherConnectSpec() { - return "tcp/" + getHost().getHostname() + ":" + getDispatchPort(); - } - - public int getDispatchPort() { - return getRelativePort(FS4_PORT); - } - private int getHttpPort() { return getRelativePort(HEALTH_PORT); } @@ -236,7 +222,7 @@ public class SearchNode extends AbstractService implements return getHostName(); } - public TransactionLogServer getTransactionLogServer() { + private TransactionLogServer getTransactionLogServer() { return tls; } @@ -269,7 +255,6 @@ public class SearchNode extends AbstractService implements @Override public void getConfig(ProtonConfig.Builder builder) { builder. - ptport(getDispatchPort()). rpcport(getRpcPort()). httpport(getHttpPort()). partition(getNodeSpec().partitionId()). diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java index fea3285e9a0..73feb0d3a16 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java @@ -17,11 +17,6 @@ public class SearchNodeWrapper implements SearchInterface { } @Override - public String getDispatcherConnectSpec() { - return node.getDispatcherConnectSpec(); - } - - @Override public String getHostName() { return node.getHostName(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java index f94ac1c285c..6fbd80adaa9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java @@ -97,4 +97,36 @@ public class SummaryTestCase { assertTrue(logger.entries.isEmpty()); } + @Test + public void testStructMemorySummary() throws ParseException { + String sd = + "search structmemorysummary {\n" + + " document structmemorysummary {\n" + + " struct elem {\n" + + " field name type string {}\n" + + " field weight type int {}\n" + + " }\n" + + " field elem_array type array<elem> {\n" + + " indexing: summary\n" + + " struct-field name {\n" + + " indexing: attribute\n" + + " }\n" + + " struct-field weight {\n" + + " indexing: attribute\n" + + " }\n" + + " }\n" + + " }\n" + + " document-summary filtered {\n" + + " summary elem_array_filtered type array<elem> {\n" + + " source: elem_array\n" + + " matched-elements-only\n" + + " }\n" + + " }\n" + + "\n" + + "}"; + DeployLoggerStub logger = new DeployLoggerStub(); + SearchBuilder.createFromString(sd, logger); + assertTrue(logger.entries.isEmpty()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 54ded910bfa..319c871f996 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -117,10 +117,6 @@ public class ClusterTest { assertEquals(19118, config.node(1).port()); assertEquals(19130, config.node(2).port()); - assertEquals(19107, config.node(0).fs4port()); - assertEquals(19119, config.node(1).fs4port()); - assertEquals(19131, config.node(2).fs4port()); - assertEquals(0, config.node(0).group()); assertEquals(0, config.node(1).group()); assertEquals(0, config.node(2).group()); diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index 5229e7da0b8..5f44b677b3b 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -59,7 +59,3 @@ node[].host string # The rpc port of this search node node[].port int - -# The legacy fs4 port of this search node -# Deprecated, will go away soon -node[].fs4port int default=0 diff --git a/container-search/CMakeLists.txt b/container-search/CMakeLists.txt index a6099fea63b..87d192a50f4 100644 --- a/container-search/CMakeLists.txt +++ b/container-search/CMakeLists.txt @@ -17,7 +17,6 @@ install_config_definition(src/main/resources/configdefinitions/query-profiles.de install_config_definition(src/main/resources/configdefinitions/rate-limiting.def search.config.rate-limiting.def) install_config_definition(src/main/resources/configdefinitions/resolvers.def search.pagetemplates.resolvers.def) install_config_definition(src/main/resources/configdefinitions/rewrites.def search.query.rewrite.rewrites.def) -install_config_definition(src/main/resources/configdefinitions/search-nodes.def search.config.dispatchprototype.search-nodes.def) install_config_definition(src/main/resources/configdefinitions/search-with-renderer-handler.def search.handler.search-with-renderer-handler.def) install_config_definition(src/main/resources/configdefinitions/searchchain-forward.def search.federation.searchchain-forward.def) install_config_definition(src/main/resources/configdefinitions/semantic-rules.def prelude.semantics.semantic-rules.def) diff --git a/container-search/src/main/resources/configdefinitions/search-nodes.def b/container-search/src/main/resources/configdefinitions/search-nodes.def deleted file mode 100644 index 84e0aa719d8..00000000000 --- a/container-search/src/main/resources/configdefinitions/search-nodes.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=search.config.dispatchprototype - -search_node[].host string -search_node[].port int diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 72f9f5b7ce2..d808b4b7adb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -39,8 +39,8 @@ public interface ZoneRegistry { /** Returns the API endpoints of all known config servers in the given zone */ List<URI> getConfigServerUris(ZoneId zoneId); - /** Returns the URI for the config server VIP in the given zone, or Optional.empty() if no VIP exists */ - default Optional<URI> getConfigServerVipUri(ZoneId zoneId) { return Optional.empty(); } + /** Returns the URI for the config server VIP in the given zone */ + URI getConfigServerVipUri(ZoneId zoneId); /** Returns all possible API endpoints of all known config servers and config server VIPs in the given zone */ List<URI> getConfigServerApiUris(ZoneId zoneId); @@ -87,4 +87,7 @@ public interface ZoneRegistry { /** Returns a URL used to generate flashy badges from strings. */ URI badgeUrl(); + /** Returns a URL to the controller's api endpoint */ + URI apiUrl(); + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 33458dbbc31..a2487e8a0d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -154,17 +154,13 @@ public class ApplicationList { } /** Returns the subset of applications which has the given upgrade policy */ - // TODO jonmv: Make this instance based when instances are orchestrated, and deployments reported per instance. public ApplicationList with(UpgradePolicy policy) { - return filteredOn(application -> application.deploymentSpec().instances().stream() - .anyMatch(instance -> instance.upgradePolicy() == policy)); + return filteredOn(application -> application.deploymentSpec().upgradePolicy() == policy); } /** Returns the subset of applications which does not have the given upgrade policy */ - // TODO jonmv: Make this instance based when instances are orchestrated, and deployments reported per instance. public ApplicationList without(UpgradePolicy policy) { - return filteredOn(application -> application.deploymentSpec().instances().stream() - .allMatch(instance -> instance.upgradePolicy() != policy)); + return filteredOn(application -> application.deploymentSpec().upgradePolicy() != policy); } /** Returns the subset of applications which have at least one deployment on a lower version than the given one */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 78ea620c4bb..9573f5d07f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -112,9 +112,8 @@ public class DeploymentTrigger { application = application.withChange(application.get().change().with(version)) .withOutstandingChange(Change.empty()); if (application.get().internal()) - for (Run run : jobs.active()) - if ( ! run.id().type().environment().isManuallyDeployed() - && TenantAndApplicationId.from(run.id().application()).equals(id)) + for (Run run : jobs.active(id)) + if ( ! run.id().type().environment().isManuallyDeployed()) jobs.abort(run.id()); } else @@ -442,7 +441,7 @@ public class DeploymentTrigger { if (jobStatus.get().lastCompleted().isEmpty()) return true; // Never completed if (jobStatus.get().firstFailing().isEmpty()) return true; // Should not happen as firstFailing should be set for an unsuccessful job if ( ! versions.targetsMatch(jobStatus.get().lastCompleted().get())) return true; // Always trigger as targets have changed - if (deploymentSpec.requireInstance(instance.name()).upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return true; // Don't throttle canaries + if (deploymentSpec.upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return true; // Don't throttle canaries Instant firstFailing = jobStatus.get().firstFailing().get().at(); Instant lastCompleted = jobStatus.get().lastCompleted().get().at(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index a4ca8605f4b..72b11f56d9f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -79,13 +79,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { ZoneId zoneId = ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion()); - // Make a local copy of the list as we want to manipulate it in case of ping problems. - List<URI> allServers = zoneRegistry.getConfigServerVipUri(zoneId) - // TODO: Use config server VIP for all zones that have one - .filter(zone -> zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) - - .map(Collections::singletonList) - .orElseGet(() -> new ArrayList<>(zoneRegistry.getConfigServerUris(zoneId))); + List<URI> allServers = getConfigserverEndpoints(zoneId); StringBuilder errorBuilder = new StringBuilder(); if (queueFirstServerIfDown(allServers, proxyRequest)) { @@ -102,6 +96,16 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { + errorBuilder.toString())); } + private List<URI> getConfigserverEndpoints(ZoneId zoneId) { + // TODO: Use config server VIP for all zones that have one + // Make a local copy of the list as we want to manipulate it in case of ping problems. + if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) { + return Collections.singletonList(zoneRegistry.getConfigServerVipUri(zoneId)); + } else { + return new ArrayList<>(zoneRegistry.getConfigServerUris(zoneId)); + } + } + private static class DiscoveryResponseStructure { public List<String> uris = new ArrayList<>(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 774c6b71434..23d2646acd7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -163,7 +163,7 @@ class JobControllerApiHandlerHelper { VespaVersion lastVespa = controller.versionStatus().version(controller.systemVersion()); VespaVersion.Confidence targetConfidence = Map.of(defaultPolicy, normal, conservative, high) - .getOrDefault(application.deploymentSpec().requireInstance(instance.name()).upgradePolicy(), broken); + .getOrDefault(application.deploymentSpec().upgradePolicy(), broken); for (VespaVersion version : controller.versionStatus().versions()) if ( ! version.versionNumber().isAfter(controller.systemVersion()) && version.confidence().equalOrHigherThan(targetConfidence)) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 4ac7ff4d6d4..2adf6ce95e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -142,7 +142,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { "/application/" + instance.id().application().value()).toString()); object.setString("upgradePolicy", toString(controller.applications().requireApplication(TenantAndApplicationId.from(instance.id())) - .deploymentSpec().requireInstance(instance.name()).upgradePolicy())); + .deploymentSpec().upgradePolicy())); } private static String toString(DeploymentSpec.UpgradePolicy upgradePolicy) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index d252b99f026..b2e2288dc51 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -15,6 +15,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; @@ -35,6 +36,7 @@ import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.JobRunnerTest; import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; +import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import javax.security.auth.x500.X500Principal; import java.math.BigInteger; @@ -102,6 +104,8 @@ public class InternalDeploymentTester { public JobRunner runner() { return runner; } public ConfigServerMock configServer() { return tester.configServer(); } public Controller controller() { return tester.controller(); } + public ControllerTester controllerTester() { return tester.controllerTester(); } + public Upgrader upgrader() { return tester.upgrader(); } public ApplicationController applications() { return tester.applications(); } public ManualClock clock() { return tester.clock(); } public Application application() { return tester.application(appId); } @@ -258,7 +262,7 @@ public class InternalDeploymentTester { .filter(r -> r.id().type() == job.type()) .orElseThrow(() -> new AssertionError(job.type() + " is not among the active: " + jobs.active())); assertFalse(run.hasFailed()); - assertNotEquals(aborted, run.status()); + assertFalse(run.hasEnded()); return run; } @@ -283,7 +287,8 @@ public class InternalDeploymentTester { if (job.type() == JobType.stagingTest) { // Do the initial deployment and installation of the real application. assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installInitialReal)); - currentRun(job).versions().sourcePlatform().ifPresent(version -> tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), version)); + Versions versions = currentRun(job).versions(); + tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), versions.sourcePlatform().orElse(versions.targetPlatform())); tester.configServer().convergeServices(id.application(), zone); setEndpoints(id.application(), zone); runner.advance(currentRun(job)); @@ -329,12 +334,7 @@ public class InternalDeploymentTester { assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installReal)); tester.configServer().convergeServices(id.application(), zone); - runner.advance(currentRun(job)); - if ( ! (currentRun(job).versions().sourceApplication().isPresent() && job.type().isProduction()) - && job.type() != JobType.stagingTest) { - assertEquals(unfinished, jobs.run(id).get().steps().get(Step.installReal)); - setEndpoints(id.application(), zone); - } + setEndpoints(id.application(), zone); runner.advance(currentRun(job)); if (job.type().environment().isManuallyDeployed()) { assertEquals(Step.Status.succeeded, jobs.run(id).get().steps().get(Step.installReal)); @@ -431,7 +431,7 @@ public class InternalDeploymentTester { /** Pulls the ready job trigger, and then runs the whole of the given job, successfully. */ public void runJob(JobId job) { - tester.readyJobTrigger().run(); + triggerJobs(); doDeploy(job); doUpgrade(job); doConverge(job); @@ -443,6 +443,22 @@ public class InternalDeploymentTester { doTeardown(job); } + public void jobAborted(JobType type) { + jobAborted(new JobId(instanceId, type)); + } + + public void jobAborted(ApplicationId instanceId, JobType type) { + jobAborted(new JobId(instanceId, type)); + } + + public void jobAborted(JobId job) { + triggerJobs(); + RunId id = currentRun(job).id(); + runner.advance(currentRun(job)); + assertEquals(aborted, jobs.run(id).get().status()); + assertTrue(jobs.run(id).get().hasEnded()); + } + public void failDeployment(JobType type) { failDeployment(new JobId(instanceId, type)); } @@ -452,8 +468,8 @@ public class InternalDeploymentTester { } public void failDeployment(JobId job) { + triggerJobs(); RunId id = currentRun(job).id(); - tester.readyJobTrigger().run(); tester.configServer().throwOnNextPrepare(new IllegalArgumentException("Exception")); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); @@ -470,8 +486,8 @@ public class InternalDeploymentTester { } public void timeOutUpgrade(JobId job) { + triggerJobs(); RunId id = currentRun(job).id(); - tester.readyJobTrigger().run(); doDeploy(job); clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1)); runner.advance(currentRun(job)); @@ -489,8 +505,8 @@ public class InternalDeploymentTester { } public void timeOutConvergence(JobId job) { + triggerJobs(); RunId id = currentRun(job).id(); - tester.readyJobTrigger().run(); doDeploy(job); doUpgrade(job); clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 7771dd2d1a0..32bbf3ceb9b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -165,6 +165,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public URI apiUrl() { + return URI.create("https://api.tld:4443/"); + } + + @Override public boolean hasZone(ZoneId zoneId) { return zones.stream().anyMatch(zone -> zone.getId().equals(zoneId)); } @@ -175,8 +180,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override - public Optional<URI> getConfigServerVipUri(ZoneId zoneId) { - return Optional.of(URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value()))); + public URI getConfigServerVipUri(ZoneId zoneId) { + return URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value())); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index 3c1fb2fffd0..9df0e99787d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -6,13 +6,16 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Before; @@ -23,6 +26,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; @@ -49,55 +53,55 @@ public class DeploymentIssueReporterTest { .upgradePolicy("canary") .build(); - - private DeploymentTester tester; + private InternalDeploymentTester tester; private DeploymentIssueReporter reporter; private MockDeploymentIssues issues; @Before public void setup() { - tester = new DeploymentTester(); + tester = new InternalDeploymentTester(); issues = new MockDeploymentIssues(); reporter = new DeploymentIssueReporter(tester.controller(), issues, Duration.ofDays(1), new JobControl(new MockCuratorDb())); } @Test public void testDeploymentFailureReporting() { - // All applications deploy from unique build projects. - Long projectId1 = 10L; - Long projectId2 = 20L; - Long projectId3 = 30L; - - Long propertyId1 = 1L; - Long propertyId2 = 2L; - Long propertyId3 = 3L; - - tester.upgradeSystem(Version.fromString("6.2")); + tester.controllerTester().upgradeSystem(Version.fromString("6.2")); - Optional<Contact> contact = Optional.of(tester.controllerTester().serviceRegistry().contactRetrieverMock().contact()); + /* tester.controllerTester().createTenant("tenant1", "domain1", 1L, contact); tester.controllerTester().createTenant("tenant2", "domain2", 1L, contact); tester.controllerTester().createTenant("tenant3", "domain3", 1L, contact); + */ // Create and deploy one application for each of three tenants. - Application app1 = tester.createApplication("application1", "tenant1", projectId1, propertyId1); - Application app2 = tester.createApplication("application2", "tenant2", projectId2, propertyId2); - Application app3 = tester.createApplication("application3", "tenant3", projectId3, propertyId3); + Application app1 = tester.createApplication("application1", "tenant1", "default"); + Application app2 = tester.createApplication("application2", "tenant2", "default"); + Application app3 = tester.createApplication("application3", "tenant3", "default"); + + Contact contact = tester.controllerTester().serviceRegistry().contactRetrieverMock().contact(); + tester.controller().tenants().lockOrThrow(app1.id().tenant(), LockedTenant.Athenz.class, tenant -> + tester.controller().tenants().store(tenant.with(contact))); + tester.controller().tenants().lockOrThrow(app2.id().tenant(), LockedTenant.Athenz.class, tenant -> + tester.controller().tenants().store(tenant.with(contact))); + tester.controller().tenants().lockOrThrow(app3.id().tenant(), LockedTenant.Athenz.class, tenant -> + tester.controller().tenants().store(tenant.with(contact))); // NOTE: All maintenance should be idempotent within a small enough time interval, so maintain is called twice in succession throughout. - // apps 1 and 3 have one failure each. - tester.jobCompletion(component).application(app1).uploadArtifact(applicationPackage).submit(); - tester.deployAndNotify(app1.id().defaultInstance(), applicationPackage, true, systemTest); - tester.deployAndNotify(app1.id().defaultInstance(), applicationPackage, false, stagingTest); + // app 1 fails staging tests. + tester.newSubmission(app1.id(), applicationPackage); + tester.runJob(app1.id().defaultInstance(), systemTest); + tester.timeOutConvergence(app1.id().defaultInstance(), stagingTest); // app2 is successful, but will fail later. - tester.deployCompletely(app2, canaryPackage); + tester.deployNewSubmission(app2.id(), tester.newSubmission(app2.id(), applicationPackage)); - tester.jobCompletion(component).application(app3).uploadArtifact(applicationPackage).submit(); - tester.deployAndNotify(app3.id().defaultInstance(), applicationPackage, true, systemTest); - tester.deployAndNotify(app3.id().defaultInstance(), applicationPackage, true, stagingTest); - tester.deployAndNotify(app3.id().defaultInstance(), applicationPackage, false, productionUsWest1); + // app 3 fails a production job. + tester.newSubmission(app3.id(), applicationPackage); + tester.runJob(app3.id().defaultInstance(), systemTest); + tester.runJob(app3.id().defaultInstance(), stagingTest); + tester.failDeployment(app3.id().defaultInstance(), productionUsWest1); reporter.maintain(); reporter.maintain(); @@ -132,7 +136,7 @@ public class DeploymentIssueReporterTest { // app3 fixes their problems, but the ticket for app3 is left open; see the resolved ticket is not escalated when another escalation period has passed. - tester.deployAndNotify(app3.id().defaultInstance(), applicationPackage, true, productionUsWest1); + tester.runJob(app3.id().defaultInstance(), productionUsWest1); tester.clock().advance(maxInactivity.plus(Duration.ofDays(1))); reporter.maintain(); @@ -143,8 +147,8 @@ public class DeploymentIssueReporterTest { // app3 now has a new failure past max failure age; see that a new issue is filed. - tester.jobCompletion(component).application(app3).nextBuildNumber().uploadArtifact(applicationPackage).submit(); - tester.deployAndNotify(app3.id().defaultInstance(), applicationPackage, false, systemTest); + tester.newSubmission(app3.id(), applicationPackage); + tester.failDeployment(app3.id().defaultInstance(), systemTest); tester.clock().advance(maxInactivity.plus(maxFailureAge)); reporter.maintain(); @@ -152,14 +156,19 @@ public class DeploymentIssueReporterTest { assertTrue("A new issue is filed for app3.", issues.isOpenFor(app3.id())); + // App2 is changed to be a canary + tester.deployNewSubmission(app2.id(), tester.newSubmission(app2.id(), canaryPackage)); + assertEquals(canary, tester.applications().requireApplication(app2.id()).deploymentSpec().requireInstance("default").upgradePolicy()); + assertEquals(Change.empty(), tester.applications().requireApplication(app2.id()).change()); + // Bump system version to upgrade canary app2. Version version = Version.fromString("6.3"); - tester.upgradeSystem(version); + tester.controllerTester().upgradeSystem(version); + tester.upgrader().maintain(); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); - tester.readyJobTrigger().maintain(); - tester.completeUpgradeWithError(app2, version, canaryPackage, systemTest); - tester.upgradeSystem(version); + tester.timeOutUpgrade(app2.id().defaultInstance(), systemTest); + tester.controllerTester().upgradeSystem(version); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); assertFalse("We have no platform issues initially.", issues.platformIssue()); @@ -171,7 +180,6 @@ public class DeploymentIssueReporterTest { reporter.maintain(); assertTrue("We get a platform issue when confidence is broken", issues.platformIssue()); assertFalse("No deployment issue is filed for app2, which has a version upgrade failure.", issues.isOpenFor(app2.id())); - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 4e60e13dd51..24420d4590a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; @@ -24,9 +23,7 @@ import org.junit.Test; import java.time.Duration; import java.util.List; -import java.util.Optional; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; @@ -77,7 +74,7 @@ public class MetricsReporterTest { @Test public void deployment_average_duration() { - DeploymentTester tester = new DeploymentTester(); + var tester = new InternalDeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") @@ -85,28 +82,30 @@ public class MetricsReporterTest { MetricsReporter reporter = createReporter(tester.controller()); - Application app = tester.createApplication("app1", "tenant1", 1, 11L); - tester.deployCompletely(app, applicationPackage); + Application app = tester.createApplication("app1", "tenant1", "default"); + tester.deployNewSubmission(app.id(), tester.newSubmission(app.id(), applicationPackage)); reporter.maintain(); assertEquals(Duration.ZERO, getAverageDeploymentDuration(app.id().defaultInstance())); // An exceptionally fast deployment :-) // App spends 3 hours deploying - tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit(); + tester.newSubmission(app.id(), applicationPackage); tester.clock().advance(Duration.ofHours(1)); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, systemTest); + tester.runJob(app.id().defaultInstance(), systemTest); tester.clock().advance(Duration.ofMinutes(30)); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, stagingTest); + tester.runJob(app.id().defaultInstance(), stagingTest); + tester.triggerJobs(); tester.clock().advance(Duration.ofMinutes(90)); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, productionUsWest1); + tester.runJob(app.id().defaultInstance(), productionUsWest1); reporter.maintain(); // Average time is 1 hour (system-test) + 90 minutes (staging-test runs in parallel with system-test) + 90 minutes (production) / 3 jobs assertEquals(Duration.ofMinutes(80), getAverageDeploymentDuration(app.id().defaultInstance())); // Another deployment starts and stalls for 12 hours - tester.jobCompletion(component).application(app).nextBuildNumber(2).uploadArtifact(applicationPackage).submit(); + tester.newSubmission(app.id(), applicationPackage); + tester.triggerJobs(); tester.clock().advance(Duration.ofHours(12)); reporter.maintain(); @@ -119,50 +118,51 @@ public class MetricsReporterTest { @Test public void deployments_failing_upgrade() { - DeploymentTester tester = new DeploymentTester(); + var tester = new InternalDeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") .build(); MetricsReporter reporter = createReporter(tester.controller()); - Application app = tester.createApplication("app1", "tenant1", 1, 11L); + Application app = tester.createApplication("app1", "tenant1", "default"); // Initial deployment without failures - tester.deployCompletely(app, applicationPackage); + tester.deployNewSubmission(app.id(), tester.newSubmission(app.id(), applicationPackage)); reporter.maintain(); assertEquals(0, getDeploymentsFailingUpgrade(app.id().defaultInstance())); // Failing application change is not counted - tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit(); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), false, systemTest); + var submission = tester.newSubmission(app.id(), applicationPackage); + tester.triggerJobs(); + tester.failDeployment(app.id().defaultInstance(), systemTest); reporter.maintain(); assertEquals(0, getDeploymentsFailingUpgrade(app.id().defaultInstance())); // Application change completes - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, systemTest); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, stagingTest); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, productionUsWest1); + tester.deployNewSubmission(app.id(), submission); assertFalse("Change deployed", tester.controller().applications().requireApplication(app.id()).change().hasTargets()); // New versions is released and upgrade fails in test environments Version version = Version.fromString("7.1"); - tester.upgradeSystem(version); + tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), false, systemTest); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), false, stagingTest); + tester.triggerJobs(); + tester.failDeployment(app.id().defaultInstance(), systemTest); + tester.failDeployment(app.id().defaultInstance(), stagingTest); reporter.maintain(); assertEquals(2, getDeploymentsFailingUpgrade(app.id().defaultInstance())); // Test and staging pass and upgrade fails in production - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, systemTest); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, stagingTest); - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), false, productionUsWest1); + tester.runJob(app.id().defaultInstance(), systemTest); + tester.runJob(app.id().defaultInstance(), stagingTest); + tester.triggerJobs(); + tester.failDeployment(app.id().defaultInstance(), productionUsWest1); reporter.maintain(); assertEquals(1, getDeploymentsFailingUpgrade(app.id().defaultInstance())); // Upgrade eventually succeeds - tester.deployAndNotify(app.id().defaultInstance(), Optional.of(applicationPackage), true, productionUsWest1); + tester.runJob(app.id().defaultInstance(), productionUsWest1); assertFalse("Upgrade deployed", tester.controller().applications().requireApplication(app.id()).change().hasTargets()); reporter.maintain(); assertEquals(0, getDeploymentsFailingUpgrade(app.id().defaultInstance())); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index f4b223ead82..69882601eed 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -172,6 +172,12 @@ public class Flags { "Takes effect on next tick or on host-admin restart.", APPLICATION_ID); + public static final UnboundStringFlag ZOOKEEPER_SERVER_VERSION = defineStringFlag( + "zookeeper-server-version", "3.4.14", + "The version of ZooKeeper server to use", + "Takes effect on restart of Docker container", + HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java index 68656f06d7d..39d56408fb8 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java @@ -42,18 +42,18 @@ public class Properties { } /** Returns the path of the API private key. This must be set with the 'privateKeyFile' property. */ - public static Path apiPrivateKeyFile() { - return Paths.get(requireNonBlankProperty("privateKeyFile")); + public static Path apiKeyFile() { + return Paths.get(requireNonBlankProperty("apiKeyFile")); } /** Returns the path of the API certificate, if this is set with the 'certificateFile' property. */ public static Optional<Path> apiCertificateFile() { - return getNonBlankProperty("certificateFile").map(Paths::get); + return getNonBlankProperty("apiCertificateFile").map(Paths::get); } /** Returns the actual private key as a string */ - public static Optional<String> privateKey() { - return getNonBlankProperty("privateKey"); + public static Optional<String> apiKey() { + return getNonBlankProperty("apiKey"); } /** Returns the path of the data plane certificate file, if this is set with the 'dataPlaneCertificateFile' property. */ @@ -61,8 +61,8 @@ public class Properties { return getNonBlankProperty("dataPlaneCertificateFile").map(Paths::get); } - /** Returns the path of the data plane private key file, if this is set with the 'dataPlanePrivateKeyFile' property. */ - public static Optional<Path> dataPlanePrivateKeyFile() { + /** Returns the path of the data plane private key file, if this is set with the 'dataPlaneKeyFile' property. */ + public static Optional<Path> dataPlaneKeyFile() { return getNonBlankProperty("dataPlaneKeyFile").map(Paths::get); } diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 0e521e473ae..26abd6be914 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -765,12 +765,14 @@ Test::requireThatAttributesAreUsed() EXPECT_EQUAL(2u, rep->docsums.size()); + // FIXME the expected output ordering of weighted set fields is currently inherently linked + // to the internal ordering of such attributes. Should be decoupled, as this is very fragile. EXPECT_TRUE(assertSlime("{ba:10,bb:10.1," "bc:'foo'," "bd:[20,30]," "be:[20.2,30.3]," "bf:['bar','baz']," - "bg:[{item:40,weight:2},{item:50,weight:3}]," + "bg:[{item:50,weight:3},{item:40,weight:2}]," "bh:[{item:40.4,weight:4},{item:50.5,weight:5}]," "bi:[{item:'quux',weight:7},{item:'qux',weight:6}]," "bj:'0x01020178017901016601674008000000000000'}", *rep, 0, true)); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index deec60dbf50..d123a5711ac 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -4,10 +4,6 @@ namespace=vespa.config.search.core ## Base directory. The default is ignored as it is assigned by the model basedir string default="." restart -## specifies the port number for the persistent internal transport -## protocol provided for a multi-level dispatch system. -ptport int default=8003 restart - ## Port to use for the rpcserver. rpcport int default=8004 restart diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index ebcbd6853cb..66ca6bd2eac 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -66,6 +66,9 @@ private: uint64_t getTotalValueCount() const override; + void apply_attribute_changes_to_array(DocumentValues& docValues); + void apply_attribute_changes_to_wset(DocumentValues& docValues); + public: void clearDocs(DocId lidLow, DocId lidLimit) override; void onShrinkLidSpace() override ; diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 2e16bf13b95..f07efcafa9a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -3,6 +3,8 @@ #pragma once #include <vespa/searchlib/attribute/multivalueattribute.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search { @@ -27,9 +29,7 @@ MultiValueAttribute(const vespalib::string &baseFileName, } template <typename B, typename M> -MultiValueAttribute<B, M>::~MultiValueAttribute() -{ -} +MultiValueAttribute<B, M>::~MultiValueAttribute() = default; template <typename B, typename M> int32_t MultiValueAttribute<B, M>::getWeight(DocId doc, uint32_t idx) const @@ -38,100 +38,162 @@ int32_t MultiValueAttribute<B, M>::getWeight(DocId doc, uint32_t idx) const return ((idx < values.size()) ? values[idx].weight() : 1); } +namespace { + +template <typename T> +struct HashFn { + using type = vespalib::hash<T>; +}; + +template <> +struct HashFn<search::datastore::EntryRef> { + struct EntryRefHasher { + size_t operator() (const search::datastore::EntryRef& v) const noexcept { + return v.ref(); + } + }; + using type = EntryRefHasher; +}; + +} template <typename B, typename M> void MultiValueAttribute<B, M>::applyAttributeChanges(DocumentValues & docValues) { + if (this->hasArrayType()) { + apply_attribute_changes_to_array(docValues); + return; + } else if (this->hasWeightedSetType()) { + apply_attribute_changes_to_wset(docValues); + return; + } +} + +template <typename B, typename M> +void +MultiValueAttribute<B, M>::apply_attribute_changes_to_array(DocumentValues& docValues) +{ // compute new values for each document with changes for (ChangeVectorIterator current(this->_changes.begin()), end(this->_changes.end()); (current != end); ) { DocId doc = current->_doc; - - MultiValueArrayRef oldValues(_mvMapping.get(doc)); - ValueVector newValues(oldValues.cbegin(), oldValues.cend()); - // find last clear doc - ChangeVectorIterator lastClearDoc = end; + ChangeVectorIterator last_clear_doc = end; for (ChangeVectorIterator iter = current; (iter != end) && (iter->_doc == doc); ++iter) { if (iter->_type == ChangeBase::CLEARDOC) { - lastClearDoc = iter; + last_clear_doc = iter; } } - // use last clear doc if found - if (lastClearDoc != end) { - current = lastClearDoc; + if (last_clear_doc != end) { + current = last_clear_doc; } + MultiValueArrayRef old_values(_mvMapping.get(doc)); + ValueVector new_values(old_values.cbegin(), old_values.cend()); + vespalib::hash_map<ValueType, size_t, typename HashFn<ValueType>::type> tombstones; // iterate through all changes for this document for (; (current != end) && (current->_doc == doc); ++current) { - if (current->_type == ChangeBase::CLEARDOC) { - newValues.clear(); + new_values.clear(); + tombstones.clear(); continue; } - ValueType data; bool hasData = extractChangeData(*current, data); - + if (!hasData) { + continue; + } if (current->_type == ChangeBase::APPEND) { - if (hasData) { - if (this->hasArrayType()) { - newValues.push_back(MultiValueType(data, current->_weight)); - } else if (this->hasWeightedSetType()) { - ValueVectorIterator witer; - for (witer = newValues.begin(); witer != newValues.end(); ++witer) { - if (witer->value() == data) { - break; - } - } - if (witer != newValues.end()) { - witer->setWeight(current->_weight); - } else { - newValues.push_back(MultiValueType(data, current->_weight)); - } - } - } + new_values.emplace_back(data, current->_weight); } else if (current->_type == ChangeBase::REMOVE) { - if (hasData) { - for (ValueVectorIterator witer = newValues.begin(); witer != newValues.end(); ) { - if (witer->value() == data) { - witer = newValues.erase(witer); - } else { - ++witer; - } - } + // Defer all removals to the very end by tracking when, during value vector build time, + // a removal was encountered for a particular value. All values < this index will be ignored. + tombstones[data] = new_values.size(); + } + } + // Optimize for the case where nothing was explicitly removed. + if (!tombstones.empty()) { + ValueVector culled; + culled.reserve(new_values.size()); + for (size_t i = 0; i < new_values.size(); ++i) { + auto iter = tombstones.find(new_values[i].value()); + if (iter == tombstones.end() || (iter->second <= i)) { + culled.emplace_back(new_values[i]); } + } + culled.swap(new_values); + } + this->checkSetMaxValueCount(new_values.size()); + docValues.emplace_back(doc, std::move(new_values)); + } +} + +template <typename B, typename M> +void +MultiValueAttribute<B, M>::apply_attribute_changes_to_wset(DocumentValues& docValues) +{ + // compute new values for each document with changes + for (ChangeVectorIterator current(this->_changes.begin()), end(this->_changes.end()); (current != end); ) { + const DocId doc = current->_doc; + // find last clear doc + ChangeVectorIterator last_clear_doc = end; + size_t max_elems_inserted = 0; + for (ChangeVectorIterator iter = current; (iter != end) && (iter->_doc == doc); ++iter) { + if (iter->_type == ChangeBase::CLEARDOC) { + last_clear_doc = iter; + } + ++max_elems_inserted; + } + // use last clear doc if found + if (last_clear_doc != end) { + current = last_clear_doc; + } + MultiValueArrayRef old_values(_mvMapping.get(doc)); + vespalib::hash_map<ValueType, int32_t, typename HashFn<ValueType>::type> wset_inserted; + wset_inserted.resize((old_values.size() + max_elems_inserted) * 2); + for (const auto& e : old_values) { + wset_inserted[e.value()] = e.weight(); + } + // iterate through all changes for this document + for (; (current != end) && (current->_doc == doc); ++current) { + if (current->_type == ChangeBase::CLEARDOC) { + wset_inserted.clear(); + continue; + } + ValueType data; + bool hasData = extractChangeData(*current, data); + if (!hasData) { + continue; + } + if (current->_type == ChangeBase::APPEND) { + wset_inserted[data] = current->_weight; + } else if (current->_type == ChangeBase::REMOVE) { + wset_inserted.erase(data); } else if ((current->_type >= ChangeBase::INCREASEWEIGHT) && (current->_type <= ChangeBase::DIVWEIGHT)) { - if (this->hasWeightedSetType() && hasData) { - ValueVectorIterator witer; - for (witer = newValues.begin(); witer != newValues.end(); ++witer) { - if (witer->value() == data) { - break; - } + auto existing = wset_inserted.find(data); + if (existing != wset_inserted.end()) { + existing->second = this->applyWeightChange(existing->second, *current); + if ((existing->second == 0) && this->getInternalCollectionType().removeIfZero()) { + wset_inserted.erase(existing); } - if (witer != newValues.end()) { - witer->setWeight(this->applyWeightChange(witer->weight(), *current)); - if (witer->weight() == 0 && this->getInternalCollectionType().removeIfZero()) { - newValues.erase(witer); - } - } else if (this->getInternalCollectionType().createIfNonExistant()) { - int32_t weight = this->applyWeightChange(0, *current); - if (weight != 0 || !this->getInternalCollectionType().removeIfZero()) { - newValues.push_back(MultiValueType(data, weight)); - } + } else if (this->getInternalCollectionType().createIfNonExistant()) { + int32_t weight = this->applyWeightChange(0, *current); + if (weight != 0 || !this->getInternalCollectionType().removeIfZero()) { + wset_inserted.insert(std::make_pair(data, weight)); } } } } - this->checkSetMaxValueCount(newValues.size()); + std::vector<MultiValueType> new_values; + new_values.reserve(wset_inserted.size()); + wset_inserted.for_each([&new_values](const auto& e){ new_values.emplace_back(e.first, e.second); }); - docValues.push_back(std::make_pair(doc, ValueVector())); - docValues.back().second.swap(newValues); + this->checkSetMaxValueCount(new_values.size()); + docValues.emplace_back(doc, std::move(new_values)); } } - template <typename B, typename M> vespalib::AddressSpace MultiValueAttribute<B, M>::getMultiValueAddressSpaceUsage() const diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java index f6a88ec83c2..c2d47622040 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java @@ -10,11 +10,11 @@ public class ApiAuthenticator implements ai.vespa.hosted.api.ApiAuthenticator { public ControllerHttpClient controller() { return Properties.apiCertificateFile() .map(certificateFile -> ControllerHttpClient.withKeyAndCertificate(Properties.apiEndpoint(), - Properties.apiPrivateKeyFile(), + Properties.apiKeyFile(), certificateFile)) .orElseGet(() -> ControllerHttpClient.withSignatureKey(Properties.apiEndpoint(), - Properties.apiPrivateKeyFile(), + Properties.apiKeyFile(), Properties.application())); } diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java index e51476907e2..37d5106d0c2 100644 --- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/EndpointAuthenticator.java @@ -50,8 +50,8 @@ public class EndpointAuthenticator implements ai.vespa.hosted.api.EndpointAuthen else { if (Properties.dataPlaneCertificateFile().isPresent()) certificateFile = Properties.dataPlaneCertificateFile().get(); - if (Properties.dataPlanePrivateKeyFile().isPresent()) - privateKeyFile = Properties.dataPlanePrivateKeyFile().get(); + if (Properties.dataPlaneKeyFile().isPresent()) + privateKeyFile = Properties.dataPlaneKeyFile().get(); } if (certificateFile != null && privateKeyFile != null) { X509Certificate certificate = X509CertificateUtils.fromPem(new String(Files.readAllBytes(certificateFile))); diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index 0698cf3c54f..6488a681c95 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -270,9 +270,13 @@ <application>${application}</application> <tenant>${tenant}</tenant> <instance>${instance}</instance> + <environment>${environment}</environment> + <region>${region}</region> <endpoint>${endpoint}</endpoint> - <privateKeyFile>${privateKeyFile}</privateKeyFile> - <certificateFile>${certificateFile}</certificateFile> + <apiKeyFile>${apiKeyFile}</apiKeyFile> + <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> + <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> + <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> </systemPropertyVariables> </configuration> </plugin> diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java index 845d0ba4c1b..55616fc7936 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -37,14 +37,14 @@ public abstract class AbstractVespaMojo extends AbstractMojo { @Parameter(property = "instance") protected String instance; - @Parameter(property = "privateKey") - protected String privateKey; + @Parameter(property = "apiKey") + protected String apiKey; - @Parameter(property = "privateKeyFile") - protected String privateKeyFile; + @Parameter(property = "apiKeyFile") + protected String apiKeyFile; - @Parameter(property = "certificateFile") - protected String certificateFile; + @Parameter(property = "apiCertificateFile") + protected String apiCertificateFile; // Fields set up as part of setup(). protected ApplicationId id; @@ -73,12 +73,12 @@ public abstract class AbstractVespaMojo extends AbstractMojo { instance = firstNonBlank(instance, project.getProperties().getProperty("instance", Properties.user())); id = ApplicationId.from(tenant, application, instance); - if (privateKey != null) { - controller = ControllerHttpClient.withSignatureKey(URI.create(endpoint), privateKey, id); - } else if (privateKeyFile != null) { - controller = certificateFile == null - ? ControllerHttpClient.withSignatureKey(URI.create(endpoint), Paths.get(privateKeyFile), id) - : ControllerHttpClient.withKeyAndCertificate(URI.create(endpoint), Paths.get(privateKeyFile), Paths.get(certificateFile)); + if (apiKey != null) { + controller = ControllerHttpClient.withSignatureKey(URI.create(endpoint), apiKey, id); + } else if (apiKeyFile != null) { + controller = apiCertificateFile == null + ? ControllerHttpClient.withSignatureKey(URI.create(endpoint), Paths.get(apiKeyFile), id) + : ControllerHttpClient.withKeyAndCertificate(URI.create(endpoint), Paths.get(apiKeyFile), Paths.get(apiCertificateFile)); } else { throw new IllegalArgumentException("One of the properties 'privateKey' or 'privateKeyFile' is required."); } |