diff options
author | Valerij Fredriksen <valerijf@verizonmedia.com> | 2020-01-16 16:27:35 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerijf@verizonmedia.com> | 2020-01-16 16:27:35 +0100 |
commit | 8b3dc39da9b25413d72eca88a78a70981017ec0e (patch) | |
tree | 69d1904e6c8f918d9c0c09ddeadc3b8bffd441fa | |
parent | 3f16a26ec4a8893ab4d01b6c6e232c8585f25f2f (diff) | |
parent | 9d3769aad649f631a652025d17325367aeaa455a (diff) |
Merge branch 'master' into freva/fallback-to-node-repo-address
# Conflicts:
# flags/src/main/java/com/yahoo/vespa/flags/Flags.java
20 files changed, 185 insertions, 201 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java index 1b8a5d14e6c..e40cb76ad7e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.text.XML; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.content.DispatchTuning; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -18,11 +19,8 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui protected Tuning doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { Tuning tuning = new Tuning(parent); for (Element e : XML.getChildren(spec)) { - if (equals("dispatch", e)) { - handleDispatch(e, tuning); - } else if (equals("searchnode", e)) { + if (equals("searchnode", e)) handleSearchNode(e, tuning); - } } return tuning; } @@ -47,15 +45,6 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui return Double.parseDouble(e.getFirstChild().getNodeValue()); } - private void handleDispatch(Element spec, Tuning t) { - t.dispatch = new Tuning.Dispatch(); - for (Element e : XML.getChildren(spec)) { - if (equals("max-hits-per-partition", e)) { - t.dispatch.maxHitsPerPartition = asInt(e); - } - } - } - private void handleSearchNode(Element spec, Tuning t) { t.searchNode = new Tuning.SearchNode(); for (Element e : XML.getChildren(spec)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index cae7ede463d..75161859068 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -266,9 +266,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private Tuning tuning; - public void setTuning(Tuning t) { - tuning = t; - } + public void setTuning(Tuning tuning) { this.tuning = tuning; } private void setResourceLimits(ResourceLimits resourceLimits) { this.resourceLimits = Optional.of(resourceLimits); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 9e5a3428726..fc470b8e560 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -2,9 +2,14 @@ package com.yahoo.vespa.model.content; /** + * Tuning of dispatching to content nodes, see the + * <a href="https://docs.vespa.ai/documentation/reference/services-content.html#dispatch-tuning">dispatch tuning documentation</a>. + * * @author Simon Thoresen Hult */ -public class TuningDispatch { +public class DispatchTuning { + + public static final DispatchTuning empty = new DispatchTuning.Builder().build(); private final Integer maxHitsPerPartition; public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE}; @@ -12,20 +17,24 @@ public class TuningDispatch { private final Double minGroupCoverage; private final Double minActiveDocsCoverage; - private TuningDispatch(Builder builder) { + private DispatchTuning(Builder builder) { maxHitsPerPartition = builder.maxHitsPerPartition; dispatchPolicy = builder.dispatchPolicy; minGroupCoverage = builder.minGroupCoverage; minActiveDocsCoverage = builder.minActiveDocsCoverage; } - public Integer getMaxHitsPerPartition() { - return maxHitsPerPartition; - } + /** Returns the max number of hits to fetch from each partition, or null to fetch all */ + public Integer maxHitsPerPartition() { return maxHitsPerPartition; } + + /** Returns the policy used to select which group to dispatch a query to */ + public DispatchPolicy dispatchPolicy() { return dispatchPolicy; } + + /** Returns the percentage of nodes in a group which must be up for that group to receive queries */ + public Double minGroupCoverage() { return minGroupCoverage; } - public DispatchPolicy getDispatchPolicy() { return dispatchPolicy; } - public Double getMinGroupCoverage() { return minGroupCoverage; } - public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; } + /** Returns the percentage of documents which must be available in a group for that group to receive queries */ + public Double minActiveDocsCoverage() { return minActiveDocsCoverage; } public static class Builder { @@ -34,8 +43,8 @@ public class TuningDispatch { private Double minGroupCoverage; private Double minActiveDocsCoverage; - public TuningDispatch build() { - return new TuningDispatch(this); + public DispatchTuning build() { + return new DispatchTuning(this); } public Builder setMaxHitsPerPartition(Integer maxHitsPerPartition) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java index 56945264789..909a29fd0a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java @@ -15,12 +15,12 @@ public class IndexedHierarchicDistributionValidator { private final String clusterName; private final StorageGroup rootGroup; private final Redundancy redundancy; - private final TuningDispatch.DispatchPolicy dispatchPolicy; + private final DispatchTuning.DispatchPolicy dispatchPolicy; public IndexedHierarchicDistributionValidator(String clusterName, StorageGroup rootGroup, Redundancy redundancy, - TuningDispatch.DispatchPolicy dispatchPolicy) { + DispatchTuning.DispatchPolicy dispatchPolicy) { this.clusterName = clusterName; this.rootGroup = rootGroup; this.redundancy = redundancy; @@ -46,7 +46,7 @@ public class IndexedHierarchicDistributionValidator { } private void validateThatLeafGroupsHasEqualNumberOfNodes() { - if (dispatchPolicy != TuningDispatch.DispatchPolicy.ROUNDROBIN) return; + if (dispatchPolicy != DispatchTuning.DispatchPolicy.ROUNDROBIN) return; StorageGroup previousGroup = null; for (StorageGroup group : rootGroup.getSubgroups()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 04e997110a6..e7a3439e6bb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -46,7 +46,6 @@ import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.content.ReservedDocumentTypeNameValidator; import com.yahoo.vespa.model.content.StorageGroup; import com.yahoo.vespa.model.content.StorageNode; -import com.yahoo.vespa.model.content.TuningDispatch; import com.yahoo.vespa.model.content.engines.PersistenceEngine; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; @@ -172,30 +171,18 @@ public class ContentCluster extends AbstractConfigProducer implements Double queryTimeout = search.getQueryTimeout(); if (queryTimeout != null) { Preconditions.checkState(index.getQueryTimeout() == null, - "You may not specify query-timeout in both proton and content."); + "In " + index + ": You may not specify query-timeout in both proton and content."); index.setQueryTimeout(queryTimeout); } Double visibilityDelay = search.getVisibilityDelay(); - if (visibilityDelay != null) { + if (visibilityDelay != null) index.setVisibilityDelay(visibilityDelay); - } index.setSearchCoverage(DomSearchCoverageBuilder.build(element)); index.setDispatchSpec(DomDispatchBuilder.build(element)); - // TODO: This should be cleaned up to avoid having to change code in 100 places - // every time we add a dispatch option. - TuningDispatch tuningDispatch = DomTuningDispatchBuilder.build(element, logger); - Integer maxHitsPerPartition = tuningDispatch.getMaxHitsPerPartition(); - if (index.getTuning() == null) index.setTuning(new Tuning(index)); - if (index.getTuning().dispatch == null) - index.getTuning().dispatch = new Tuning.Dispatch(); - if (maxHitsPerPartition != null) - index.getTuning().dispatch.maxHitsPerPartition = maxHitsPerPartition; - index.getTuning().dispatch.minGroupCoverage = tuningDispatch.getMinGroupCoverage(); - index.getTuning().dispatch.minActiveDocsCoverage = tuningDispatch.getMinActiveDocsCoverage(); - index.getTuning().dispatch.policy = tuningDispatch.getDispatchPolicy(); + index.getTuning().dispatch = DomTuningDispatchBuilder.build(element, logger); } private void setupDocumentProcessing(ContentCluster c, ModelElement e) { @@ -650,7 +637,7 @@ public class ContentCluster extends AbstractConfigProducer implements super.validate(); if (search.usesHierarchicDistribution() && !isHosted) { // validate manually configured groups - new IndexedHierarchicDistributionValidator(search.getClusterName(), rootGroup, redundancy, search.getIndexed().getTuning().dispatch.policy).validate(); + new IndexedHierarchicDistributionValidator(search.getClusterName(), rootGroup, redundancy, search.getIndexed().getTuning().dispatch.dispatchPolicy()).validate(); } new ReservedDocumentTypeNameValidator().validate(documentDefinitions); new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java index 536793e848f..b53d66632a8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.vespa.model.content.TuningDispatch; +import com.yahoo.vespa.model.content.DispatchTuning; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import java.util.logging.Level; @@ -12,8 +12,8 @@ import java.util.logging.Level; */ public class DomTuningDispatchBuilder { - public static TuningDispatch build(ModelElement contentXml, DeployLogger logger) { - TuningDispatch.Builder builder = new TuningDispatch.Builder(); + public static DispatchTuning build(ModelElement contentXml, DeployLogger logger) { + DispatchTuning.Builder builder = new DispatchTuning.Builder(); ModelElement tuningElement = contentXml.child("tuning"); if (tuningElement == null) { return builder.build(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 85663a59556..8a88e720bed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -86,6 +86,9 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer public abstract void getConfig(RankProfilesConfig.Builder builder); public abstract void getConfig(AttributesConfig.Builder builder); + @Override + public String toString() { return "search-capable cluster '" + clusterName + "'"; } + public static final class IndexingMode { public static final IndexingMode REALTIME = new IndexingMode("REALTIME"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 53b4d085351..78cdb53871d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -307,12 +307,12 @@ public class IndexedSearchCluster extends SearchCluster if (useAdaptiveDispatch) builder.distributionPolicy(DistributionPolicy.ADAPTIVE); - if (tuning.dispatch.minActiveDocsCoverage != null) - builder.minActivedocsPercentage(tuning.dispatch.minActiveDocsCoverage); - if (tuning.dispatch.minGroupCoverage != null) - builder.minGroupCoverage(tuning.dispatch.minGroupCoverage); - if (tuning.dispatch.policy != null) { - switch (tuning.dispatch.policy) { + if (tuning.dispatch.minActiveDocsCoverage() != null) + builder.minActivedocsPercentage(tuning.dispatch.minActiveDocsCoverage()); + if (tuning.dispatch.minGroupCoverage() != null) + builder.minGroupCoverage(tuning.dispatch.minGroupCoverage()); + if (tuning.dispatch.dispatchPolicy() != null) { + switch (tuning.dispatch.dispatchPolicy()) { case ADAPTIVE: builder.distributionPolicy(DistributionPolicy.ADAPTIVE); break; @@ -321,8 +321,8 @@ public class IndexedSearchCluster extends SearchCluster break; } } - if (tuning.dispatch.maxHitsPerPartition != null) - builder.maxHitsPerNode(tuning.dispatch.maxHitsPerPartition); + if (tuning.dispatch.maxHitsPerPartition() != null) + builder.maxHitsPerNode(tuning.dispatch.maxHitsPerPartition()); builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); builder.searchableCopies(rootDispatch.getSearchableCopies()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 95498a2a902..f5a91297e9e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.model.content.TuningDispatch; +import com.yahoo.vespa.model.content.DispatchTuning; import static com.yahoo.text.Lowercase.toLowerCase; @@ -15,14 +15,6 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Producer { - public static class Dispatch { - - public Integer maxHitsPerPartition = null; - public TuningDispatch.DispatchPolicy policy = null; - public Double minGroupCoverage = null; - public Double minActiveDocsCoverage = null; - } - public static class SearchNode implements ProtonConfig.Producer { public enum IoType { @@ -390,7 +382,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ } } - public Dispatch dispatch; + public DispatchTuning dispatch = DispatchTuning.empty; public SearchNode searchNode; public Tuning(AbstractConfigProducer parent) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index b39e503f837..060fff5bf66 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.CollectionUtil; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.vespa.model.content.DispatchTuning; import com.yahoo.vespa.model.search.Tuning; import org.junit.Test; import org.w3c.dom.Element; @@ -46,33 +47,6 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { } @Test - public void requireThatNullDispatchIsSafe() { - Tuning tuning = newTuning("<tuning />"); - assertNull(tuning.dispatch); - } - - @Test - public void requireThatEmptyDispatchIsSafe() { - Tuning tuning = newTuning("<tuning><dispatch/></tuning>"); - Tuning.Dispatch dispatch = tuning.dispatch; - assertNotNull(dispatch); - assertNull(dispatch.maxHitsPerPartition); - } - - @Test - public void requireThatDispatchSettingsAreParsed() { - Tuning tuning = createTuning(parse("<tuning>" + - " <dispatch>" + - " <max-hits-per-partition>69</max-hits-per-partition>" + - " </dispatch>" + - "</tuning>")); - Tuning.Dispatch dispatch = tuning.dispatch; - assertNotNull(dispatch); - assertNotNull(dispatch.maxHitsPerPartition); - assertEquals(69, dispatch.maxHitsPerPartition.intValue()); - } - - @Test public void requireThatWeCanParseRequestThreadsTag() { Tuning t = createTuning(parseXml("<requestthreads>", "<search>123</search>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java new file mode 100644 index 00000000000..4d71e0ef568 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java @@ -0,0 +1,57 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * @author Simon Thoresen Hult + */ +public class DispatchTuningTest { + + @Test + public void requireThatAccessorWork() { + DispatchTuning dispatch = new DispatchTuning.Builder() + .setMaxHitsPerPartition(69) + .setDispatchPolicy("round-robin") + .setMinGroupCoverage(7.5) + .setMinActiveDocsCoverage(12.5) + .build(); + assertEquals(69, dispatch.maxHitsPerPartition().intValue()); + assertEquals(7.5, dispatch.minGroupCoverage().doubleValue(), 0.0); + assertEquals(12.5, dispatch.minActiveDocsCoverage().doubleValue(), 0.0); + assertTrue(DispatchTuning.DispatchPolicy.ROUNDROBIN == dispatch.dispatchPolicy()); + } + @Test + public void requireThatRandomDispatchWork() { + DispatchTuning dispatch = new DispatchTuning.Builder() + .setDispatchPolicy("random") + .build(); + assertTrue(DispatchTuning.DispatchPolicy.ADAPTIVE == dispatch.dispatchPolicy()); + assertNull(dispatch.minGroupCoverage()); + assertNull(dispatch.minActiveDocsCoverage()); + } + + @Test + public void requireThatWeightedDispatchWork() { + DispatchTuning dispatch = new DispatchTuning.Builder() + .setDispatchPolicy("adaptive") + .build(); + assertTrue(DispatchTuning.DispatchPolicy.ADAPTIVE == dispatch.dispatchPolicy()); + assertNull(dispatch.minGroupCoverage()); + assertNull(dispatch.minActiveDocsCoverage()); + } + + @Test + public void requireThatDefaultsAreNull() { + DispatchTuning dispatch = new DispatchTuning.Builder().build(); + assertNull(dispatch.maxHitsPerPartition()); + assertNull(dispatch.dispatchPolicy()); + assertNull(dispatch.minActiveDocsCoverage()); + assertNull(dispatch.minGroupCoverage()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java deleted file mode 100644 index fe00648317a..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/TuningDispatchTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.content; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * @author Simon Thoresen Hult - */ -public class TuningDispatchTest { - - @Test - public void requireThatAccessorWork() { - TuningDispatch dispatch = new TuningDispatch.Builder() - .setMaxHitsPerPartition(69) - .setDispatchPolicy("round-robin") - .setMinGroupCoverage(7.5) - .setMinActiveDocsCoverage(12.5) - .build(); - assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); - assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0); - assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0); - assertTrue(TuningDispatch.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); - } - @Test - public void requireThatRandomDispatchWork() { - TuningDispatch dispatch = new TuningDispatch.Builder() - .setDispatchPolicy("random") - .build(); - assertTrue(TuningDispatch.DispatchPolicy.ADAPTIVE == dispatch.getDispatchPolicy()); - assertNull(dispatch.getMinGroupCoverage()); - assertNull(dispatch.getMinActiveDocsCoverage()); - } - - @Test - public void requireThatWeightedDispatchWork() { - TuningDispatch dispatch = new TuningDispatch.Builder() - .setDispatchPolicy("adaptive") - .build(); - assertTrue(TuningDispatch.DispatchPolicy.ADAPTIVE == dispatch.getDispatchPolicy()); - assertNull(dispatch.getMinGroupCoverage()); - assertNull(dispatch.getMinActiveDocsCoverage()); - } - - @Test - public void requireThatDefaultsAreNull() { - TuningDispatch dispatch = new TuningDispatch.Builder().build(); - assertNull(dispatch.getMaxHitsPerPartition()); - assertNull(dispatch.getDispatchPolicy()); - } -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java index 0c7c16c15c7..3f3d4b5e4c1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.cluster; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import com.yahoo.vespa.model.content.TuningDispatch; +import com.yahoo.vespa.model.content.DispatchTuning; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; @@ -18,41 +17,41 @@ import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult */ -public class DomTuningDispatchBuilderTest { +public class DomDispatchTuningBuilderTest { @Test public void requireThatDefaultsAreNull() throws Exception { - TuningDispatch dispatch = newTuningDispatch( + DispatchTuning dispatch = newTuningDispatch( "<content/>"); - assertNull(dispatch.getMaxHitsPerPartition()); + assertNull(dispatch.maxHitsPerPartition()); } @Test public void requireThatEmptyTuningIsSafe() throws Exception { - TuningDispatch dispatch = newTuningDispatch( + DispatchTuning dispatch = newTuningDispatch( "<content>" + " <tuning/>" + "</content>"); - assertNull(dispatch.getMaxHitsPerPartition()); + assertNull(dispatch.maxHitsPerPartition()); } @Test public void requireThatEmptydispatchIsSafe() throws Exception { - TuningDispatch dispatch = newTuningDispatch( + DispatchTuning dispatch = newTuningDispatch( "<content>" + " <tuning>" + " <dispatch/>" + " </tuning>" + "</content>"); - assertNull(dispatch.getMaxHitsPerPartition()); - assertNull(dispatch.getMinGroupCoverage()); - assertNull(dispatch.getMinActiveDocsCoverage()); - assertNull(dispatch.getDispatchPolicy()); + assertNull(dispatch.maxHitsPerPartition()); + assertNull(dispatch.minGroupCoverage()); + assertNull(dispatch.minActiveDocsCoverage()); + assertNull(dispatch.dispatchPolicy()); } @Test public void requireThatTuningDispatchCanBeBuilt() throws Exception { - TuningDispatch dispatch = newTuningDispatch( + DispatchTuning dispatch = newTuningDispatch( "<content>" + " <tuning>" + " <dispatch>" + @@ -62,13 +61,13 @@ public class DomTuningDispatchBuilderTest { " </dispatch>" + " </tuning>" + "</content>"); - assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); - assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0); - assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0); + assertEquals(69, dispatch.maxHitsPerPartition().intValue()); + assertEquals(7.5, dispatch.minGroupCoverage().doubleValue(), 0.0); + assertEquals(12.5, dispatch.minActiveDocsCoverage().doubleValue(), 0.0); } @Test public void requireThatTuningDispatchPolicyRoundRobin() throws Exception { - TuningDispatch dispatch = newTuningDispatch( + DispatchTuning dispatch = newTuningDispatch( "<content>" + " <tuning>" + " <dispatch>" + @@ -76,11 +75,11 @@ public class DomTuningDispatchBuilderTest { " </dispatch>" + " </tuning>" + "</content>"); - assertTrue(TuningDispatch.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); + assertTrue(DispatchTuning.DispatchPolicy.ROUNDROBIN == dispatch.dispatchPolicy()); } @Test public void requireThatTuningDispatchPolicyRandom() throws Exception { - TuningDispatch dispatch = newTuningDispatch( + DispatchTuning dispatch = newTuningDispatch( "<content>" + " <tuning>" + " <dispatch>" + @@ -88,10 +87,10 @@ public class DomTuningDispatchBuilderTest { " </dispatch>" + " </tuning>" + "</content>"); - assertTrue(TuningDispatch.DispatchPolicy.ADAPTIVE == dispatch.getDispatchPolicy()); + assertTrue(DispatchTuning.DispatchPolicy.ADAPTIVE == dispatch.dispatchPolicy()); } - private static TuningDispatch newTuningDispatch(String xml) throws Exception { + private static DispatchTuning newTuningDispatch(String xml) throws Exception { return DomTuningDispatchBuilder.build( new ModelElement(DocumentBuilderFactory.newInstance() .newDocumentBuilder() diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 33c568061fe..8d5e7015947 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,6 +7,7 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true +tenants{}.applications{}.use4443Upstream bool default=false tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 43b78212827..35c41711a6e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -759,12 +759,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private String getLogServerURI(ApplicationId applicationId, Optional<String> hostname) { // Allow to get logs from a given hostname if the application is under the hosted-vespa tenant. // We make no validation that the hostname is actually allocated to the given application since - // most applications under hosted-vespa are not known to the model and its OK for a user to get + // most applications under hosted-vespa are not known to the model and it's OK for a user to get // logs for any host if they are authorized for the hosted-vespa tenant. if (hostname.isPresent()) { if (HOSTED_VESPA_TENANT.equals(applicationId.tenant())) return "http://" + hostname.get() + ":8080/logs"; - else throw new IllegalArgumentException("Only hostname paramater unsupported for application " + applicationId); + else + throw new IllegalArgumentException("Using hostname parameter when getting logs is not supported for application " + + applicationId); } Application application = getApplication(applicationId); @@ -773,7 +775,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye HostInfo logServerHostInfo = hostInfos.stream() .filter(host -> host.getServices().stream() .anyMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver"))) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find HostInfo for LogServer")); + .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find host info for logserver")); ServiceInfo serviceInfo = logServerHostInfo.getServices().stream().filter(service -> List.of(LOGSERVER_CONTAINER.serviceName, CONTAINER.serviceName).contains(service.getServiceType())) .findFirst().orElseThrow(() -> new IllegalArgumentException("No container running on logserver host")); @@ -782,11 +784,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private int servicePort(ServiceInfo serviceInfo) { - int port = serviceInfo.getPorts().stream() + return serviceInfo.getPorts().stream() .filter(portInfo -> portInfo.getTags().stream().anyMatch(tag -> tag.equalsIgnoreCase("http"))) .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find HTTP port")) .getPort(); - return port; } public Slime createDeployLog() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index 0f5642525d3..6549f359b29 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -4,13 +4,10 @@ package com.yahoo.vespa.config.server.http; import ai.vespa.util.http.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.yolean.Exceptions; -import org.apache.http.Header; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import java.io.IOException; -import java.io.OutputStream; -import java.util.Optional; import java.util.logging.Logger; /** @@ -31,26 +28,4 @@ public class LogRetriever { } } - private static class ProxyResponse extends HttpResponse { - - private final org.apache.http.HttpResponse clientResponse; - - private ProxyResponse(org.apache.http.HttpResponse clientResponse) { - super(clientResponse.getStatusLine().getStatusCode()); - this.clientResponse = clientResponse; - } - - @Override - public String getContentType() { - return Optional.ofNullable(clientResponse.getFirstHeader("Content-Type")) - .map(Header::getValue) - .orElseGet(super::getContentType); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - clientResponse.getEntity().writeTo(outputStream); - } - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java new file mode 100644 index 00000000000..4c310dd8a0d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ProxyResponse.java @@ -0,0 +1,36 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpResponse; +import org.apache.http.Header; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Optional; + +/** + * Proxies response back to client, keeps Content-Type header if it is present + * + * @author Ola Aunrønning + */ +class ProxyResponse extends HttpResponse { + + private final org.apache.http.HttpResponse clientResponse; + + ProxyResponse(org.apache.http.HttpResponse clientResponse) { + super(clientResponse.getStatusLine().getStatusCode()); + this.clientResponse = clientResponse; + } + + @Override + public String getContentType() { + return Optional.ofNullable(clientResponse.getFirstHeader("Content-Type")) + .map(Header::getValue) + .orElseGet(super::getContentType); + } + + @Override + public void render(OutputStream outputStream) throws IOException { + clientResponse.getEntity().writeTo(outputStream); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 5dd2c8fd58a..d88fae0a8ef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,7 +9,10 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -32,10 +35,12 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; + private final BooleanFlag use4443Upstream; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; + this.use4443Upstream = Flags.USE_4443_UPSTREAM.bindTo(flagSource); } @Override @@ -62,6 +67,8 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); + ab.use4443Upstream( + use4443Upstream.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value()); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index 09ad715b471..2f70c37cd48 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -82,7 +82,10 @@ public class Node { } @Override - public String toString() { return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + - " statusIsKnown = " + statusIsKnown.get() + " working = " + working.get() + " activeDocs = " + activeDocuments.get(); } + public String toString() { + return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + + " statusIsKnown = " + statusIsKnown.get() + " working = " + working.get() + + " activeDocs = " + activeDocuments.get(); + } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 729f5fe8707..265e829c97e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -190,12 +190,19 @@ public class Flags { "Takes effect on config server restart", ZONE_ID); + public static final UnboundBooleanFlag USE_4443_UPSTREAM = defineFeatureFlag( + "use-4443-upstream", false, + "Use port 4443 for nginx upstream", + "Takes effect when routing container asks for new config", + APPLICATION_ID); + public static final UnboundBooleanFlag FAIL_STARTING_NODE_ON_IP_MISMATCH = defineFeatureFlag( "fail-starting-node-on-ip-mismatch", false, "Whether node-admin should refuse to start container when there is an IP mismatch between the DNS and node-repository", "Takes effect on next node creation (f.ex. node reboot or vespa version upgrade)", ZONE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { |