diff options
251 files changed, 5641 insertions, 3874 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index 163834a78b1..6cdcc17400c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -26,7 +26,7 @@ else fi mvn_install() { - mvn --quiet --batch-mode --no-snapshot-updates clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true "$@" + mvn --quiet --batch-mode --no-snapshot-updates clean install -Dmaven.javadoc.skip=true "$@" } # Generate vtag map diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 55260b6e68f..dd2ffba20ec 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -290,14 +290,16 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, if (dataType instanceof CollectionDataType) { dataType = ((CollectionDataType)dataType).getNestedType(); } - SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; - if (subType == null) { - throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'."); - } - for (Field field : subType.fieldSet()) { - SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), - isHeader, subType, new Matching(), true, recursion + 1); - structFields.put(field.getName(), subField); + if (dataType instanceof StructDataType) { + SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; + if (subType == null) { + throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'."); + } + for (Field field : subType.fieldSet()) { + SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), + isHeader, subType, new Matching(), true, recursion + 1); + structFields.put(field.getName(), subField); + } } } } @@ -305,41 +307,43 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, public void populateWithStructMatching(SDDocumentType sdoc, String name, DataType dataType, Matching superFieldMatching) { DataType dt = getFirstStructOrMapRecursive(); - if (dt != null) { - if (dataType instanceof MapDataType) { - MapDataType mdt = (MapDataType) dataType; - - Matching keyFieldMatching = new Matching(); - if (superFieldMatching != null) { - keyFieldMatching.merge(superFieldMatching); - } - SDField keyField = structFields.get(name.concat(".key")); - if (keyField != null) { - keyField.populateWithStructMatching(sdoc, name.concat(".key"), mdt.getKeyType(), keyFieldMatching); - keyField.setMatching(keyFieldMatching); - } + if (dt == null) { + return; + } + if (dataType instanceof MapDataType) { + MapDataType mdt = (MapDataType) dataType; - Matching valueFieldMatching = new Matching(); - if (superFieldMatching != null) { - valueFieldMatching.merge(superFieldMatching); - } - SDField valueField = structFields.get(name.concat(".value")); - if (valueField != null) { - valueField.populateWithStructMatching(sdoc, name.concat(".value"), mdt.getValueType(), - valueFieldMatching); - valueField.setMatching(valueFieldMatching); - } + Matching keyFieldMatching = new Matching(); + if (superFieldMatching != null) { + keyFieldMatching.merge(superFieldMatching); + } + SDField keyField = structFields.get(name.concat(".key")); + if (keyField != null) { + keyField.populateWithStructMatching(sdoc, name.concat(".key"), mdt.getKeyType(), keyFieldMatching); + keyField.setMatching(keyFieldMatching); + } - } else { + Matching valueFieldMatching = new Matching(); + if (superFieldMatching != null) { + valueFieldMatching.merge(superFieldMatching); + } + SDField valueField = structFields.get(name.concat(".value")); + if (valueField != null) { + valueField.populateWithStructMatching(sdoc, name.concat(".value"), mdt.getValueType(), + valueFieldMatching); + valueField.setMatching(valueFieldMatching); + } - if (dataType instanceof CollectionDataType) { - dataType = ((CollectionDataType)dataType).getNestedType(); - } + } else { + if (dataType instanceof CollectionDataType) { + dataType = ((CollectionDataType)dataType).getNestedType(); + } + if (dataType instanceof StructDataType) { SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; if (subType != null) { for (Field f : subType.fieldSet()) { if (f instanceof SDField) { - SDField field = (SDField)f; + SDField field = (SDField) f; Matching subFieldMatching = new Matching(); if (superFieldMatching != null) { subFieldMatching.merge(superFieldMatching); @@ -348,11 +352,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, SDField subField = structFields.get(field.getName()); if (subField != null) { subField.populateWithStructMatching(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), - subFieldMatching); + subFieldMatching); subField.setMatching(subFieldMatching); } } else { - throw new IllegalArgumentException("Field in struct is not SDField " + f.getName()); + throw new IllegalArgumentException("Field in struct is not SDField " + f.getName()); } } } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 9fafe89ea54..1f6f7ad6c69 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -44,6 +44,7 @@ import java.util.logging.Logger; public class VespaDomBuilder extends VespaModelBuilder { public static final String JVMARGS_ATTRIB_NAME = "jvmargs"; + public static final String GCOPTS_ATTRIB_NAME = "gcopts"; public static final String PRELOAD_ATTRIB_NAME = "preload"; // Intended for vespa engineers public static final String MMAP_NOCORE_LIMIT = "mmap-core-limit"; // Intended for vespa engineers public static final String CORE_ON_OOM = "core-on-oom"; // Intended for vespa engineers diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 502df074ec4..ccd828b5f48 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -14,6 +14,9 @@ import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.container.BundlesConfig; import com.yahoo.container.ComponentsConfig; @@ -140,6 +143,8 @@ public final class ContainerCluster public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler"; public static final String STATISTICS_HANDLER_CLASS = "com.yahoo.container.config.StatisticsRequestHandler"; public static final String SIMPLE_LINGUISTICS_PROVIDER = "com.yahoo.language.provider.SimpleLinguisticsProvider"; + public static final String CMS = "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1"; + static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15"; public static final String ROOT_HANDLER_BINDING = "*://*/"; @@ -181,6 +186,7 @@ public final class ContainerCluster private Zone zone; private String hostClusterId = null; + private String gcopts = null; private Integer memoryPercentage = null; private static class AcceptAllVerifier implements ContainerClusterVerifier { @@ -625,9 +631,33 @@ public final class ContainerCluster if (containerSearch!=null) containerSearch.getConfig(builder); } + private String buildGCOpts(Zone zone) { + Optional<String> gcopts = getGCOpts(); + if (gcopts.isPresent()) { + return gcopts.get(); + } else if (zone.system() == SystemName.dev) { + return G1GC; + } else if (isHostedVespa()) { + return ((zone.environment() != Environment.prod) || RegionName.from("us-east-3").equals(zone.region())) + ? G1GC : CMS; + } else { + return CMS; + } + } + @Override public void getConfig(QrStartConfig.Builder builder) { - if (containerSearch!=null) containerSearch.getConfig(builder); + QrStartConfig.Jvm.Builder jvmBuilder = new QrStartConfig.Jvm.Builder(); + if (getMemoryPercentage().isPresent()) { + jvmBuilder.heapSizeAsPercentageOfPhysicalMemory(getMemoryPercentage().get()); + } else if (isHostedVespa()) { + jvmBuilder.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ? 17 : 60); + } + if (containerSearch!=null) { + jvmBuilder.directMemorySizeCache(containerSearch.totalCacheSizeMb()); + } + jvmBuilder.gcopts(buildGCOpts(getZone())); + builder.jvm(jvmBuilder); } @Override @@ -784,6 +814,8 @@ public final class ContainerCluster public Optional<String> getHostClusterId() { return Optional.ofNullable(hostClusterId); } public void setMemoryPercentage(Integer memoryPercentage) { this.memoryPercentage = memoryPercentage; } + public void setGCOpts(String gcopts) { this.gcopts = gcopts; } + public Optional<String> getGCOpts() { return Optional.ofNullable(gcopts); } /** * Returns the percentage of host physical memory this application has specified for nodes in this cluster, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 2cfd4b067cd..3a7b7864554 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.container.search.searchchain.HttpProvider; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.container.QrSearchersConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; @@ -36,7 +35,6 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> IndexInfoConfig.Producer, IlscriptsConfig.Producer, QrSearchersConfig.Producer, - QrStartConfig.Producer, QueryProfilesConfig.Producer, SemanticRulesConfig.Producer, PageTemplatesConfig.Producer { @@ -47,13 +45,12 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; - private final ContainerCluster owningCluster; public ContainerSearch(ContainerCluster cluster, SearchChains chains, Options options) { super(chains); this.options = options; - this.owningCluster = cluster; cluster.addComponent(getFS4ResourcePool()); + } private Component<?, ComponentModel> getFS4ResourcePool() { @@ -113,18 +110,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> if (pageTemplates!=null) pageTemplates.getConfig(builder); } - @Override - public void getConfig(QrStartConfig.Builder qsB) { - QrStartConfig.Jvm.Builder internalBuilder = new QrStartConfig.Jvm.Builder(); - if (owningCluster.getMemoryPercentage().isPresent()) { - internalBuilder.heapSizeAsPercentageOfPhysicalMemory(owningCluster.getMemoryPercentage().get()); - } else if (owningCluster.isHostedVespa()) { - internalBuilder.heapSizeAsPercentageOfPhysicalMemory(owningCluster.getHostClusterId().isPresent() ? 17 : 60); - } - qsB.jvm(internalBuilder.directMemorySizeCache(totalCacheSizeMb())); - } - - private int totalCacheSizeMb() { + public int totalCacheSizeMb() { return totalHttpProviderCacheSize(); } @@ -192,6 +178,6 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> * Struct that encapsulates qrserver options. */ public static class Options { - public Map<String, QrsCache> cacheSettings = new LinkedHashMap<>(); + Map<String, QrsCache> cacheSettings = new LinkedHashMap<>(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 89f2995d179..ac1f313d983 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -76,6 +76,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -449,6 +451,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addContainers(Collections.singleton(container)); } + static boolean incompatibleGCOptions(String jvmargs) { + Pattern gcAlgorithm = Pattern.compile("-XX:[-+]Use.+GC"); + Pattern cmsArgs = Pattern.compile("-XX:[-+]*CMS"); + return (gcAlgorithm.matcher(jvmargs).find() ||cmsArgs.matcher(jvmargs).find()); + } private void addNodesFromXml(ContainerCluster cluster, Element containerElement, ConfigModelContext context) { Element nodesElement = XML.getChild(containerElement, "nodes"); if (nodesElement == null) { // default single node on localhost @@ -460,7 +467,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } else { List<Container> nodes = createNodes(cluster, nodesElement, context); - applyNodesTagJvmArgs(nodes, nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)); + String jvmArgs = nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); + String gcopts = nodesElement.hasAttribute(VespaDomBuilder.GCOPTS_ATTRIB_NAME) + ? nodesElement.getAttribute(VespaDomBuilder.GCOPTS_ATTRIB_NAME) + : null; + if (incompatibleGCOptions(jvmArgs)) { + context.getDeployLogger().log(Level.WARNING, "You need to move out your GC related options from 'jvmargs' to 'gcopts'"); + } else { + cluster.setGCOpts(gcopts); + } + applyNodesTagJvmArgs(nodes, jvmArgs); applyRoutingAliasProperties(nodes, cluster); applyDefaultPreload(nodes, nodesElement); applyMemoryPercentage(cluster, nodesElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); @@ -659,10 +675,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return false; } - private void applyNodesTagJvmArgs(List<Container> containers, String nodesTagJvnArgs) { + private void applyNodesTagJvmArgs(List<Container> containers, String jvmArgs) { for (Container container: containers) { if (container.getAssignedJvmArgs().isEmpty()) - container.prependJvmArgs(nodesTagJvnArgs); + container.prependJvmArgs(jvmArgs); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 985b6e1e4b0..623a963f77a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -3,16 +3,17 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.config.search.DispatchConfig; -import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.log.LogLevel; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.searchdefinition.DocumentOnlySearch; import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.DispatchConfig; +import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.SimpleConfigProducer; @@ -23,8 +24,11 @@ import com.yahoo.vespa.model.content.DispatchSpec; import com.yahoo.vespa.model.content.SearchCoverage; import java.io.File; -import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import java.util.logging.Logger; /** @@ -316,17 +320,17 @@ public class IndexedSearchCluster extends SearchCluster @Override public DerivedConfiguration getSdConfig() { return null; } - + @Override public void getConfig(IndexInfoConfig.Builder builder) { unionCfg.getConfig(builder); } - + @Override public void getConfig(IlscriptsConfig.Builder builder) { unionCfg.getConfig(builder); } - + @Override public void getConfig(AttributesConfig.Builder builder) { unionCfg.getConfig(builder); @@ -402,6 +406,19 @@ public class IndexedSearchCluster extends SearchCluster nodeBuilder.fs4port(node.getDispatchPort()); if (tuning.dispatch.minActiveDocsCoverage != null) builder.minActivedocsPercentage(tuning.dispatch.minActiveDocsCoverage); + if (tuning.dispatch.minGroupCoverage != null) + builder.minGroupCoverage(tuning.dispatch.minGroupCoverage); + if (tuning.dispatch.policy != null) { + switch (tuning.dispatch.policy) { + case RANDOM: + builder.distributionPolicy(DistributionPolicy.RANDOM); + break; + case ROUNDROBIN: + builder.distributionPolicy(DistributionPolicy.ROUNDROBIN); + break; + } + } + builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); builder.node(nodeBuilder); } } diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index 85963764139..6a82556f01b 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -2,6 +2,7 @@ service.attlist &= attribute hostalias { xsd:NCName } service.attlist &= attribute baseport { xsd:unsignedShort }? service.attlist &= attribute jvmargs { text }? +service.attlist &= attribute gcopts { text }? # preload is for internal use only service.attlist &= attribute preload { text }? diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 4934ce113bb..4862fdf7a50 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -211,6 +211,7 @@ DocumentApi = element document-api { NodesOfContainerCluster = element nodes { attribute jvmargs { text }? & + attribute gcopts { text }? & attribute preload { text }? & attribute allocated-memory { text }? & attribute cpu-socket-affinity { xsd:boolean }? & diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 4a9a6d3dff3..8396cac265c 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -82,12 +82,11 @@ public class ModelProvisioningTest { " <nodes count=\"3\"/>" + "</jdisc>" + "<jdisc id='mydisc2' version='1.0'>" + - " <search/>" + " <document-processing/>" + " <handler id='myHandler'>" + " <component id='injected' />" + " </handler>" + - " <nodes count='2' allocated-memory='45%' jvmargs='-verbosegc' preload='lib/blablamalloc.so'/>" + + " <nodes count='2' allocated-memory='45%' gcopts='-XX:+UseParNewGC' jvmargs='-verbosegc' preload='lib/blablamalloc.so'/>" + "</jdisc>" + "</services>"; String hosts ="<hosts>" @@ -112,35 +111,38 @@ public class ModelProvisioningTest { + "</hosts>"; VespaModelCreatorWithMockPkg creator = new VespaModelCreatorWithMockPkg(null, services); VespaModel model = creator.create(new DeployState.Builder().modelHostProvisioner(new InMemoryProvisioner(Hosts.readFrom(new StringReader(hosts)), true))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().size(), is(3)); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getConfigId(), is("mydisc/container.0")); - assertTrue(model.getContainerClusters().get("mydisc").getContainers().get(0).isInitialized()); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getConfigId(), is("mydisc/container.1")); - assertTrue(model.getContainerClusters().get("mydisc").getContainers().get(1).isInitialized()); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getConfigId(), is("mydisc/container.2")); - assertTrue(model.getContainerClusters().get("mydisc").getContainers().get(2).isInitialized()); - - assertThat(model.getContainerClusters().get("mydisc2").getContainers().size(), is(2)); - assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getConfigId(), is("mydisc2/container.0")); - assertTrue(model.getContainerClusters().get("mydisc2").getContainers().get(0).isInitialized()); - assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(1).getConfigId(), is("mydisc2/container.1")); - assertTrue(model.getContainerClusters().get("mydisc2").getContainers().get(1).isInitialized()); - - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getJvmArgs(), is("")); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getJvmArgs(), is("")); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getJvmArgs(), is("")); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); - assertThat(model.getContainerClusters().get("mydisc").getMemoryPercentage(), is(Optional.empty())); - - assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getJvmArgs(), is("-verbosegc")); - assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(1).getJvmArgs(), is("-verbosegc")); - assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getPreLoad(), is("lib/blablamalloc.so")); - assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(1).getPreLoad(), is("lib/blablamalloc.so")); - assertThat(model.getContainerClusters().get("mydisc2").getMemoryPercentage(), is(Optional.of(45))); + ContainerCluster mydisc = model.getContainerClusters().get("mydisc"); + ContainerCluster mydisc2 = model.getContainerClusters().get("mydisc2"); + assertThat(mydisc.getContainers().size(), is(3)); + assertThat(mydisc.getContainers().get(0).getConfigId(), is("mydisc/container.0")); + assertTrue(mydisc.getContainers().get(0).isInitialized()); + assertThat(mydisc.getContainers().get(1).getConfigId(), is("mydisc/container.1")); + assertTrue(mydisc.getContainers().get(1).isInitialized()); + assertThat(mydisc.getContainers().get(2).getConfigId(), is("mydisc/container.2")); + assertTrue(mydisc.getContainers().get(2).isInitialized()); + + assertThat(mydisc2.getContainers().size(), is(2)); + assertThat(mydisc2.getContainers().get(0).getConfigId(), is("mydisc2/container.0")); + assertTrue(mydisc2.getContainers().get(0).isInitialized()); + assertThat(mydisc2.getContainers().get(1).getConfigId(), is("mydisc2/container.1")); + assertTrue(mydisc2.getContainers().get(1).isInitialized()); + + assertThat(mydisc.getContainers().get(0).getJvmArgs(), is("")); + assertThat(mydisc.getContainers().get(1).getJvmArgs(), is("")); + assertThat(mydisc.getContainers().get(2).getJvmArgs(), is("")); + assertThat(mydisc.getContainers().get(0).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); + assertThat(mydisc.getContainers().get(1).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); + assertThat(mydisc.getContainers().get(2).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); + assertThat(mydisc.getMemoryPercentage(), is(Optional.empty())); + + assertThat(mydisc2.getContainers().get(0).getJvmArgs(), is("-verbosegc")); + assertThat(mydisc2.getContainers().get(1).getJvmArgs(), is("-verbosegc")); + assertThat(mydisc2.getContainers().get(0).getPreLoad(), is("lib/blablamalloc.so")); + assertThat(mydisc2.getContainers().get(1).getPreLoad(), is("lib/blablamalloc.so")); + assertThat(mydisc2.getMemoryPercentage(), is(Optional.of(45))); + assertThat(mydisc2.getGCOpts(), is(Optional.of("-XX:+UseParNewGC"))); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); - model.getContainerClusters().get("mydisc2").getConfig(qrStartBuilder); + mydisc2.getConfig(qrStartBuilder); QrStartConfig qrsStartConfig = new QrStartConfig(qrStartBuilder); assertEquals(45, qrsStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); @@ -1663,7 +1665,7 @@ public class ModelProvisioningTest { private int physicalMemoryPercentage(ContainerCluster cluster) { QrStartConfig.Builder b = new QrStartConfig.Builder(); - cluster.getSearch().getConfig(b); + cluster.getConfig(b); return new QrStartConfig(b).jvm().heapSizeAsPercentageOfPhysicalMemory(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java index d6e31ac8934..675e04191f8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java @@ -26,6 +26,11 @@ public class DisallowComplexMapAndWsetKeyTypesTestCase { testFieldType("array<map<mystruct,string>>"); } + @Test + public void requireThatNestedComplexValuesForMapSucceed() throws ParseException { + testFieldType("array<map<string,mystruct>>"); + } + @Test(expected = IllegalArgumentException.class) public void requireThatNestedComplexTypesForWsetFail() throws ParseException { testFieldType("array<weightedset<mystruct>>"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 8e096b14d85..e4fb19010f5 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -107,6 +107,10 @@ public class ContainerClusterTest { DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(isHosted).build()).build(); return new MockRoot("foo", state); } + private MockRoot createRoot(boolean isHosted, Zone zone) { + DeployState state = new DeployState.Builder().zone(zone).properties(new DeployProperties.Builder().hostedVespa(isHosted).build()).build(); + return new MockRoot("foo", state); + } private ContainerCluster createContainerCluster(MockRoot root, boolean isCombinedCluster, Integer memoryPercentage, Optional<ContainerClusterVerifier> extraComponents) { @@ -124,7 +128,7 @@ public class ContainerClusterTest { int expectedMemoryPercentage) { ContainerCluster cluster = createContainerCluster(createRoot(isHosted), isCombinedCluster, explicitMemoryPercentage); QrStartConfig.Builder qsB = new QrStartConfig.Builder(); - cluster.getSearch().getConfig(qsB); + cluster.getConfig(qsB); QrStartConfig qsC= new QrStartConfig(qsB); assertEquals(expectedMemoryPercentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory()); } @@ -154,6 +158,7 @@ public class ContainerClusterTest { assertEquals(expectedArgs, jvmArgs); } } + private void verifyJvmArgs(boolean isHosted, boolean hasDocProc) { MockRoot root = createRoot(isHosted); ContainerCluster cluster = createContainerCluster(root, false); @@ -174,6 +179,36 @@ public class ContainerClusterTest { verifyJvmArgs(isHosted, hasDocProc, "", container.getJvmArgs()); } + private void verifyGCOpts(boolean isHosted, String override, Zone zone, String expected) { + MockRoot root = createRoot(isHosted, zone); + ContainerCluster cluster = createContainerCluster(root, false); + addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + cluster.setGCOpts(override); + assertEquals(1, cluster.getContainers().size()); + QrStartConfig.Builder qsB = new QrStartConfig.Builder(); + cluster.getConfig(qsB); + QrStartConfig qsC= new QrStartConfig(qsB); + assertEquals(expected, qsC.jvm().gcopts()); + } + + private void verifyGCOpts(boolean isHosted, Zone zone, String expected) { + verifyGCOpts(isHosted, null, zone, expected); + verifyGCOpts(isHosted, "-XX:+UseG1GC", zone, "-XX:+UseG1GC"); + Zone DEV = new Zone(SystemName.dev, zone.environment(), zone.region()); + verifyGCOpts(isHosted, null, DEV, ContainerCluster.G1GC); + verifyGCOpts(isHosted, "-XX:+UseConcMarkSweepGC", DEV, "-XX:+UseConcMarkSweepGC"); + + } + + @Test + public void requireThatGCOptsIsHonoured() { + final Zone US_EAST_3 = new Zone(Environment.prod, RegionName.from("us-east-3")); + verifyGCOpts(false, Zone.defaultZone(),ContainerCluster.CMS); + verifyGCOpts(false, US_EAST_3, ContainerCluster.CMS); + verifyGCOpts(true, Zone.defaultZone(), ContainerCluster.CMS); + verifyGCOpts(true, US_EAST_3, ContainerCluster.G1GC); + } + @Test public void testContainerClusterMaxThreads() { MockRoot root = createRoot(false); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index f94ebab42a9..a3e24b8a520 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -28,6 +28,7 @@ import com.yahoo.container.usability.BindingsOverviewHandler; import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.net.HostName; import com.yahoo.prelude.cluster.QrMonitorConfig; +import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.Container; @@ -68,6 +69,46 @@ import static org.junit.Assert.fail; public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test + public void detect_conflicting_gcoptions_in_jvmargs() { + assertFalse(ContainerModelBuilder.incompatibleGCOptions("")); + assertFalse(ContainerModelBuilder.incompatibleGCOptions("UseG1GC")); + assertTrue(ContainerModelBuilder.incompatibleGCOptions("-XX:+UseG1GC")); + assertTrue(ContainerModelBuilder.incompatibleGCOptions("abc -XX:+UseParNewGC xyz")); + assertTrue(ContainerModelBuilder.incompatibleGCOptions("-XX:CMSInitiatingOccupancyFraction=19")); + } + + @Test + public void honours_gcopts() { + Element clusterElem = DomBuilderTest.parse( + "<jdisc version='1.0'>", + " <search/>", + " <nodes gcopts='-XX:+UseG1GC'>", + " <node hostalias='mockhost'/>", + " </nodes>", + "</jdisc>" ); + createModel(root, clusterElem); + QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); + root.getConfig(qrStartBuilder, "jdisc/container.0"); + QrStartConfig qrStartConfig = new QrStartConfig(qrStartBuilder); + assertEquals("-XX:+UseG1GC", qrStartConfig.jvm().gcopts()); + } + + @Test + public void ignores_gcopts_on_conflicting_jvargs() { + Element clusterElem = DomBuilderTest.parse( + "<jdisc version='1.0'>", + " <nodes gcopts='-XX:+UseG1GC' jvmargs='-XX:+UseParNewGC'>", + " <node hostalias='mockhost'/>", + " </nodes>", + "</jdisc>" ); + createModel(root, clusterElem); + QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); + root.getConfig(qrStartBuilder, "jdisc/container.0"); + QrStartConfig qrStartConfig = new QrStartConfig(qrStartBuilder); + assertEquals(ContainerCluster.CMS, qrStartConfig.jvm().gcopts()); + } + + @Test public void default_port_is_4080() throws Exception { Element clusterElem = DomBuilderTest.parse( "<jdisc version='1.0'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index f4d3fbc782c..5acfe9312f6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -217,7 +217,7 @@ public class DocprocBuilderTest extends DomBuilderTest { QrStartConfig.Jvm jvm = qrStartConfig.jvm(); assertThat(jvm.server(), is(true)); assertThat(jvm.verbosegc(), is(true)); - assertThat(jvm.gcopts(), is("")); + assertThat(jvm.gcopts(), is("-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1")); assertThat(jvm.heapsize(), is(1536)); assertThat(jvm.stacksize(), is(512)); assertThat(qrStartConfig.ulimitv(), is("")); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index 122d2bf18e3..dbc57dd5abd 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -9,7 +9,6 @@ import com.yahoo.cloud.config.ApplicationIdConfig; * @author Ulf Lilleengen * @author vegard * @author bratseth - * @since 5.1 */ public final class ApplicationId implements Comparable<ApplicationId> { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index d56a7c3e298..44bd7ec3708 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -9,43 +9,53 @@ package com.yahoo.config.provision; public enum NodeType { /** A node to be assigned to a tenant to run application workloads */ - tenant(false, "Tenant node"), + tenant(null, "Tenant node"), /** A host of a set of (Docker) tenant nodes */ - host(true, "Tenant docker host"), + host(tenant, "Tenant docker host"), /** Nodes running the shared proxy layer */ - proxy(false, "Proxy node"), + proxy(null, "Proxy node"), /** A host of a (Docker) proxy node */ - proxyhost(true, "Proxy docker host"), + proxyhost(proxy, "Proxy docker host"), /** A config server */ - config(false, "Config server"), + config(null, "Config server"), /** A host of a (Docker) config server node */ - confighost(true, "Config docker host"), + confighost(config, "Config docker host"), /** A controller */ - controller(false, "Controller"), + controller(null, "Controller"), /** A host of a (Docker) controller node */ - controllerhost(true, "Controller host"); + controllerhost(controller, "Controller host"); - private final boolean isDockerHost; + private final NodeType childNodeType; private final String description; - NodeType(boolean isDockerHost, String description) { - this.isDockerHost = isDockerHost; + NodeType(NodeType childNodeType, String description) { + this.childNodeType = childNodeType; this.description = description; } public boolean isDockerHost() { - return isDockerHost; + return childNodeType != null; } public String description() { return description; } + /** + * @return {@link NodeType} of the node(s) that run on this host + * @throws IllegalStateException if this type is not a host + */ + public NodeType childNodeType() { + if (! isDockerHost()) + throw new IllegalStateException(this + " has no children"); + + return childNodeType; + } } diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index d8ef600a33f..602d3b17a8e 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -7,6 +7,15 @@ namespace=vespa.config.search # for that group to be included in queries minActivedocsPercentage double default=97.0 +# Minimum coverage for allowing a group to be considered for serving +minGroupCoverage double default=100 + +# Maximum number of nodes allowed to be down for group to be considered for serving +maxNodesDownPerGroup int default=0 + +# Distribution policy for group selection +distributionPolicy enum { ROUNDROBIN, RANDOM } default=ROUNDROBIN + # The unique key of a search node node[].key int diff --git a/configserver/pom.xml b/configserver/pom.xml index 2aeecb1ba2f..a9cade47446 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -47,6 +47,18 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>orchestrator</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application-model</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-bundle</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3e11637b801..0594524af37 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -53,6 +53,7 @@ import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; import java.io.IOException; @@ -94,31 +95,36 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final DeployLogger logger = new SilentDeployLogger(); private final ConfigserverConfig configserverConfig; private final FileDistributionStatus fileDistributionStatus; + private final Orchestrator orchestrator; @Inject public ApplicationRepository(TenantRepository tenantRepository, HostProvisionerProvider hostProvisionerProvider, ConfigConvergenceChecker configConvergenceChecker, HttpProxy httpProxy, - ConfigserverConfig configserverConfig) { + ConfigserverConfig configserverConfig, + Orchestrator orchestrator) { this(tenantRepository, hostProvisionerProvider.getHostProvisioner(), - configConvergenceChecker, httpProxy, configserverConfig, Clock.systemUTC(), new FileDistributionStatus()); + configConvergenceChecker, httpProxy, configserverConfig, orchestrator, + Clock.systemUTC(), new FileDistributionStatus()); } // For testing public ApplicationRepository(TenantRepository tenantRepository, Provisioner hostProvisioner, + Orchestrator orchestrator, Clock clock) { - this(tenantRepository, hostProvisioner, clock, new ConfigserverConfig(new ConfigserverConfig.Builder())); + this(tenantRepository, hostProvisioner, orchestrator, clock, new ConfigserverConfig(new ConfigserverConfig.Builder())); } // For testing public ApplicationRepository(TenantRepository tenantRepository, Provisioner hostProvisioner, + Orchestrator orchestrator, Clock clock, ConfigserverConfig configserverConfig) { this(tenantRepository, Optional.of(hostProvisioner), new ConfigConvergenceChecker(), new HttpProxy(new SimpleHttpFetcher()), - configserverConfig, clock, new FileDistributionStatus()); + configserverConfig, orchestrator, clock, new FileDistributionStatus()); } private ApplicationRepository(TenantRepository tenantRepository, @@ -126,6 +132,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ConfigConvergenceChecker configConvergenceChecker, HttpProxy httpProxy, ConfigserverConfig configserverConfig, + Orchestrator orchestrator, Clock clock, FileDistributionStatus fileDistributionStatus) { this.tenantRepository = tenantRepository; @@ -134,6 +141,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.httpProxy = httpProxy; this.clock = clock; this.configserverConfig = configserverConfig; + this.orchestrator = orchestrator; this.fileDistributionStatus = fileDistributionStatus; } @@ -387,6 +395,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye hostProvisioner.ifPresent(provisioner -> provisioner.restart(applicationId, hostFilter)); } + public boolean isSuspended(ApplicationId application) { + return orchestrator.getAllSuspendedApplications().contains(application); + } + public HttpResponse filedistributionStatus(ApplicationId applicationId, Duration timeout) { return fileDistributionStatus.status(getApplication(applicationId), timeout); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index a3c17f3a89e..be99212c176 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -28,9 +28,9 @@ import java.time.Duration; * Operations on applications (delete, wait for config convergence, restart, application content etc.) * * @author hmusum - * @since 5.4 */ public class ApplicationHandler extends HttpHandler { + private final Zone zone; private final ApplicationRepository applicationRepository; @@ -102,6 +102,10 @@ public class ApplicationHandler extends HttpHandler { return applicationRepository.getLogs(applicationId, apiParams); } + if (isIsSuspendedRequest(request)) { + return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId)); + } + return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); } @@ -142,6 +146,7 @@ public class ApplicationHandler extends HttpHandler { "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*", @@ -150,6 +155,11 @@ public class ApplicationHandler extends HttpHandler { "http://*/application/v2/tenant/*/application/*"); } + private static boolean isIsSuspendedRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() == 7 && + request.getUri().getPath().endsWith("/suspended"); + } + private static boolean isLogRequest(HttpRequest request) { return getBindingMatch(request).groupCount() == 4 && request.getUri().getPath().endsWith("/logs"); @@ -228,4 +238,12 @@ public class ApplicationHandler extends HttpHandler { object.setLong("generation", generation); } } + + private static class ApplicationSuspendedResponse extends JSONResponse { + ApplicationSuspendedResponse(boolean suspended) { + super(Response.Status.OK); + object.setBool("suspended", suspended); + } + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java index 15401709f1c..56895e3516e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java @@ -23,9 +23,9 @@ import java.util.List; * Handler for listing currently active applications for a tenant. * * @author Ulf Lilleengen - * @since 5.1 */ public class ListApplicationsHandler extends HttpHandler { + private final TenantRepository tenantRepository; private final Zone zone; @@ -40,7 +40,7 @@ public class ListApplicationsHandler extends HttpHandler { @Override public HttpResponse handleGET(HttpRequest request) { TenantName tenantName = Utils.getTenantNameFromApplicationsRequest(request); - final String urlBase = Utils.getUrlBase(request, "/application/v2/tenant/" + tenantName + "/application/"); + String urlBase = Utils.getUrlBase(request, "/application/v2/tenant/" + tenantName + "/application/"); List<ApplicationId> applicationIds = listApplicationIds(tenantName); Collection<String> applicationUrls = Collections2.transform(applicationIds, new Function<ApplicationId, String>() { @@ -67,4 +67,5 @@ public class ListApplicationsHandler extends HttpHandler { sb.append("/instance/").append(id.instance().value()); return sb.toString(); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java index 3089216f433..a4527305abb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java @@ -15,10 +15,11 @@ import java.util.Collection; * Response that lists applications. * * @author Ulf Lilleengen - * @since 5.1 */ public class ListApplicationsResponse extends HttpResponse { + private final Slime slime = new Slime(); + public ListApplicationsResponse(int status, Collection<String> applications) { super(status); Cursor array = slime.setArray(); @@ -36,4 +37,5 @@ public class ListApplicationsResponse extends HttpResponse { public String getContentType() { return HttpConfigResponse.JSON_CONTENT_TYPE; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java index 4eefcc0ca75..95a71881b47 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * Tenant list response * * @author vegardh - * */ public class ListTenantsResponse extends SessionResponse { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java index 99320df11df..bb2b57ba45c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java @@ -32,4 +32,5 @@ public class PrepareResult { public Slime deployLog() { return deployLog; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index e482bec9ed9..2ccef8b85df 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.config.server.http.Utils; * Handler that activates a session given by tenant and id (PUT). * * @author vegardh - * @since 5.1 */ public class SessionActiveHandler extends SessionHandler { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java index 15faf267cce..94137f58a39 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java @@ -18,17 +18,16 @@ import com.yahoo.vespa.config.server.http.Utils; * in the session's application package * * @author Ulf Lilleengen - * @since 5.1 */ public class SessionContentHandler extends SessionHandler { + private final TenantRepository tenantRepository; private final ContentHandler contentHandler = new ContentHandler(); @Inject public SessionContentHandler(SessionHandler.Context ctx, ApplicationRepository applicationRepository, - TenantRepository tenantRepository) - { + TenantRepository tenantRepository) { super(ctx, applicationRepository); this.tenantRepository = tenantRepository; } @@ -53,7 +52,7 @@ public class SessionContentHandler extends SessionHandler { } private SessionContentRequestV2 getContentRequest(HttpRequest request) { - final TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); + TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); validateRequest(tenantName); long sessionId = getSessionIdV2(request); String contentPath = SessionContentRequestV2.getContentPath(request); diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 60dd7b0cea2..8521ca8b31f 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -133,6 +133,7 @@ <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding> <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding> + <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge</binding> @@ -192,5 +193,7 @@ <preprocess:include file='configserver-config.xml' required='false' /> <preprocess:include file='configserver-components.xml' required='false' /> + + <preprocess:include file='zookeeper-server-config.xml' required='false' /> </jdisc> </services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index ef25effbc64..694464ee578 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -19,6 +19,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.v2.PrepareResult; @@ -76,6 +77,7 @@ public class ApplicationRepositoryTest { private ApplicationRepository applicationRepository; private TenantRepository tenantRepository; private SessionHandlerTest.MockProvisioner provisioner; + private OrchestratorMock orchestrator; private TimeoutBudget timeoutBudget; @Rule @@ -90,8 +92,9 @@ public class ApplicationRepositoryTest { tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); tenantRepository.addTenant(tenant3); + orchestrator = new OrchestratorMock(); provisioner = new SessionHandlerTest.MockProvisioner(); - applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, orchestrator, clock); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); } @@ -118,6 +121,14 @@ public class ApplicationRepositoryTest { } @Test + public void testSuspension() { + deployApp(testApp); + assertFalse(applicationRepository.isSuspended(applicationId())); + orchestrator.suspend(applicationId()); + assertTrue(applicationRepository.isSuspended(applicationId())); + } + + @Test public void getLogs() { WireMockServer wireMock = new WireMockServer(wireMockConfig().port(8080)); wireMock.start(); @@ -197,7 +208,7 @@ public class ApplicationRepositoryTest { tenantRepository.addTenant(tenant1); Provisioner provisioner = new SessionHandlerTest.MockProvisioner(); - applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, orchestrator, clock); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); // TODO: Deploy an app with a bundle or file that will be a file reference, too much missing in test setup to get this working now diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java new file mode 100644 index 00000000000..e61d3710fac --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java @@ -0,0 +1,83 @@ +package com.yahoo.vespa.config.server.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.orchestrator.Host; +import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.model.NodeGroup; +import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostStatus; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * (Only the suspended applications part of this is in use) + * + * @author bratseth + */ +public class OrchestratorMock implements Orchestrator { + + private final Set<HostName> suspendedHosts = new HashSet<>(); + private final Set<ApplicationId> suspendedApplications = new HashSet<>(); + + @Override + public Host getHost(HostName hostName) { + return null; + } + + @Override + public HostStatus getNodeStatus(HostName hostName) { + return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS; + } + + @Override + public void setNodeStatus(HostName hostName, HostStatus state) {} + + @Override + public void resume(HostName hostName) { + suspendedHosts.remove(hostName); + } + + @Override + public void suspend(HostName hostName) { + suspendedHosts.add(hostName); + } + + @Override + public void suspendGroup(NodeGroup nodeGroup) { + nodeGroup.getHostNames().forEach(this::suspend); + } + + @Override + public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationId appId) { + return suspendedApplications.contains(appId) + ? ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN : ApplicationInstanceStatus.NO_REMARKS; + } + + @Override + public Set<ApplicationId> getAllSuspendedApplications() { + return Collections.unmodifiableSet(suspendedApplications); + } + + @Override + public void resume(ApplicationId appId) { + suspendedApplications.remove(appId); + } + + @Override + public void suspend(ApplicationId appId) { + suspendedApplications.add(appId); + } + + @Override + public void acquirePermissionToRemove(HostName hostName) {} + + @Override + public void suspendAll(HostName parentHostname, List<HostName> hostNames) { + hostNames.forEach(this::suspend); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 0d2654d693e..952d757be53 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -29,6 +29,7 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; @@ -122,7 +123,11 @@ public class DeployTester { catch (Exception e) { throw new IllegalArgumentException(e); } - applicationRepository = new ApplicationRepository(tenantRepository, new ProvisionerAdapter(provisioner), clock, configserverConfig); + applicationRepository = new ApplicationRepository(tenantRepository, + new ProvisionerAdapter(provisioner), + new OrchestratorMock(), + clock, + configserverConfig); } public Tenant tenant() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java index 5097ed1d67c..0e2cee24b60 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java @@ -13,7 +13,6 @@ import static org.junit.Assert.*; * Base class for handler tests * * @author hmusum - * @since 5.1.14 */ public class HandlerTest { @@ -49,4 +48,5 @@ public class HandlerTest { public static void assertHttpStatusCodeAndMessage(HttpResponse response, int statusCode, String contentType, String message) throws IOException { assertHttpStatusCodeErrorCodeAndMessage(response, statusCode, null, contentType, message); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/LogRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/LogRetrieverTest.java index 738903910f3..a2c63047878 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/LogRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/LogRetrieverTest.java @@ -21,11 +21,10 @@ import static org.junit.Assert.assertEquals; public class LogRetrieverTest { - private String logServerHostName = "http://localhost:8080/"; private LogRetriever logRetriever; @Rule - public final WireMockRule wireMock = new WireMockRule(options().port(8080), true); + public final WireMockRule wireMock = new WireMockRule(options().dynamicPort(), true); @Before public void setup() { @@ -36,7 +35,8 @@ public class LogRetrieverTest { public void testThatLogHandlerPropagatesResponseBody() throws IOException { String expectedBody = "{logs-json}"; stubFor(get(urlEqualTo("/")).willReturn(okJson(expectedBody))); - HttpResponse response = logRetriever.getLogs(logServerHostName); + String logServerUri = "http://localhost:" + wireMock.port() +"/"; + HttpResponse response = logRetriever.getLogs(logServerUri); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); response.render(byteArrayOutputStream); assertEquals(expectedBody, byteArrayOutputStream.toString()); @@ -44,9 +44,9 @@ public class LogRetrieverTest { } @Test - public void testThatNotFoundLogServerReturns404() throws IOException { + public void testThatNotFoundLogServerReturns404() { stubFor(get(urlEqualTo("/")).willReturn(aResponse().withStatus(200))); - HttpResponse response = logRetriever.getLogs("http://wrong-host:8080/"); + HttpResponse response = logRetriever.getLogs("http://wrong-host:" + wireMock.port() + "/"); assertEquals(404, response.getStatus()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 221d134c0f5..b0bb3bf244f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -10,6 +10,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -66,6 +67,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { Zone.defaultZone(), new ApplicationRepository(tenantRepository, new MockProvisioner(), + new OrchestratorMock(), clock)); pathPrefix = createPath(idTenant1, Zone.defaultZone()); baseUrl = baseServer + pathPrefix; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index c8c1815bba0..2c84e2d8ad4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.HttpProxy; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.StaticResponse; @@ -59,10 +60,12 @@ public class ApplicationHandlerTest { private final static TenantName mytenantName = TenantName.from("mytenant"); private final static TenantName foobar = TenantName.from("foobar"); private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); + private TenantRepository tenantRepository; private ApplicationRepository applicationRepository; private SessionHandlerTest.MockProvisioner provisioner; private MockStateApiFactory stateApiFactory = new MockStateApiFactory(); + private OrchestratorMock orchestrator; @Before public void setup() { @@ -71,8 +74,11 @@ public class ApplicationHandlerTest { tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName)); tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); provisioner = new SessionHandlerTest.MockProvisioner(); + orchestrator = new OrchestratorMock(); applicationRepository = new ApplicationRepository(tenantRepository, - provisioner, Clock.systemUTC()); + provisioner, + orchestrator, + Clock.systemUTC()); listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), tenantRepository, Zone.defaultZone()); @@ -135,6 +141,14 @@ public class ApplicationHandlerTest { } @Test + public void testSuspended() throws Exception { + applicationRepository.deploy(testApp, prepareParams(applicationId)); + assertSuspended(false, applicationId, Zone.defaultZone()); + orchestrator.suspend(applicationId); + assertSuspended(true, applicationId, Zone.defaultZone()); + } + + @Test public void testConverge() throws Exception { applicationRepository.deploy(testApp, prepareParams(applicationId)); converge(applicationId, Zone.defaultZone()); @@ -150,7 +164,8 @@ public class ApplicationHandlerTest { HostProvisionerProvider.withProvisioner(provisioner), new ConfigConvergenceChecker(stateApiFactory), mockHttpProxy, - new ConfigserverConfig(new ConfigserverConfig.Builder())); + new ConfigserverConfig(new ConfigserverConfig.Builder()), + new OrchestratorMock()); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); when(mockHttpProxy.get(any(), eq(host), eq("container-clustercontroller"), eq("clustercontroller-status/v1/clusterName1"))) .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); @@ -237,6 +252,12 @@ public class ApplicationHandlerTest { assertApplicationGeneration(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedGeneration); } + private void assertSuspended(boolean expectedValue, ApplicationId application, Zone zone) throws IOException { + String restartUrl = toUrlPath(application, zone, true) + "/suspended"; + HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "{\"suspended\":" + expectedValue + "}"); + } + private String toUrlPath(ApplicationId application, Zone zone, boolean fullAppIdInUrl) { String url = "http://myhost:14000/application/v2/tenant/" + application.tenant().value() + "/application/" + application.application().value(); if (fullAppIdInUrl) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index b4e3f2374be..aab5fc68d1d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.SuperModelGenerationCounter; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; @@ -360,7 +361,10 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { private SessionActiveHandler createHandler() { return new SessionActiveHandler(SessionActiveHandler.testOnlyContext(), - new ApplicationRepository(tenantRepository, hostProvisioner, clock), + new ApplicationRepository(tenantRepository, + hostProvisioner, + new OrchestratorMock(), + clock), tenantRepository, Zone.defaultZone()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 1428e384f2b..42b3fadc0de 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -10,6 +10,7 @@ import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantBuilder; @@ -169,7 +170,10 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { private SessionContentHandler createHandler() { return new SessionContentHandler( SessionContentHandler.testOnlyContext(), - new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), clock), + new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + clock), tenantRepository); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 640f5dafbed..803a87ada1c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -9,6 +9,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest; import com.yahoo.vespa.config.server.http.HandlerTest; @@ -227,6 +228,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { SessionCreateHandler.testOnlyContext(), new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), clock), tenantRepository, componentRegistry.getConfigserverConfig()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index e9b53c95c70..e8b20217a76 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -21,6 +21,7 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; @@ -385,6 +386,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { SessionPrepareHandler.testOnlyContext(), new ApplicationRepository(tenantRepository, new MockProvisioner(), + new OrchestratorMock(), clock), tenantRepository, componentRegistry.getConfigserverConfig()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index 35b22d19d6a..a4117ee7b51 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -14,6 +14,7 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.SessionResponse; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -38,7 +39,10 @@ public class TenantHandlerTest { public void setup() { tenantRepository = new TenantRepository(new TestComponentRegistry.Builder().curator(new MockCurator()).build()); ApplicationRepository applicationRepository = - new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), Clock.systemUTC()); + new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); handler = new TenantHandler(TenantHandler.testOnlyContext(), tenantRepository, applicationRepository); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index b92feffbb55..659baf5a184 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; @@ -21,7 +22,10 @@ class MaintainerTester { curator = new MockCurator(); GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); tenantRepository = new TenantRepository(componentRegistry, false); - applicationRepository = new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), Clock.systemUTC()); + applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); } Curator curator() { return curator; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 7ad59d66916..76ea4579244 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -4,7 +4,7 @@ package com.yahoo.container.handler; import org.json.JSONException; import org.json.JSONObject; -import javax.xml.bind.DatatypeConverter; +import java.util.Base64; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -29,7 +29,7 @@ public class LogReader { for(File child : files) { long logTime = Files.readAttributes(child.toPath(), BasicFileAttributes.class).creationTime().toMillis(); if(child.isFile() && earliestLogThreshold < logTime && logTime < latestLogThreshold) { - json.put(filename + child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath()))); + json.put(filename + child.getName(), Base64.getEncoder().encodeToString(Files.readAllBytes(child.toPath()))); } else if (!child.isFile()){ traverse_folder(child, json, filename + child.getName() + "-"); diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 42f1014a6c1..8dc7c2685bf 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -13,7 +13,15 @@ import com.yahoo.jdisc.References; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.SharedResource; import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.ConfigAgent; +import com.yahoo.messagebus.DestinationSessionParams; +import com.yahoo.messagebus.DynamicThrottlePolicy; +import com.yahoo.messagebus.IntermediateSessionParams; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.Protocol; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.messagebus.ThrottlePolicy; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.shared.SharedDestinationSession; diff --git a/container-search-gui/pom.xml b/container-search-gui/pom.xml index 12f425f8b4b..1804e029039 100644 --- a/container-search-gui/pom.xml +++ b/container-search-gui/pom.xml @@ -58,7 +58,9 @@ </dependency> </dependencies> <build> + <plugins> + <!-- TODO: alternative solution. For now, don't call out during build. <plugin> <groupId>com.googlecode.maven-download-plugin</groupId> <artifactId>download-maven-plugin</artifactId> @@ -77,6 +79,7 @@ </execution> </executions> </plugin> + --> <plugin> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin</artifactId> diff --git a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html new file mode 100644 index 00000000000..0ae9e72a19b --- /dev/null +++ b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html @@ -0,0 +1,1914 @@ + +<!DOCTYPE html> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<html lang="en"> + +<head> + <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <!-- <meta name="viewport" content="width=device-width, initial-scale=1"> --> + <meta name="description" content="Serve big data with ease - dynamic responses at any scale in milliseconds with any traffic volume."> + <meta name="author" content="Vespa team"> + + <title>Search API Reference</title> + + <!-- Bootstrap --> + <link href="/css/bootstrap.min.css" rel="stylesheet"> + + <!-- Font Awesome --> + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <!-- Docs layout --> + <link rel="stylesheet" href="/css/docs.css"> + + <!-- Favicons --> + <link rel="icon" href="/icons/favicon.ico" type="image/x-icon" /> + <link rel="shortcut icon" href="/icons/favicon.ico" type="image/x-icon" /> + + <!-- Bootstrap and other JavaScript --> + <script type="text/javascript" async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-MML-AM_CHTML"></script> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script> + <script src="/js/bootstrap.min.js"></script> + + <!-- Global Site Tag (gtag.js) - Google Analytics --> + <script async src="https://www.googletagmanager.com/gtag/js?id=UA-107187180-3"></script> + <script> + window.dataLayer = window.dataLayer || []; + function gtag(){dataLayer.push(arguments)}; + gtag('js', new Date()); + gtag('config', 'UA-107187180-3'); +</script> + +</head> + +<body> + +<!-- Fixed navbar --> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<nav class="navbar navbar-default navbar-fixed-top"> + <div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://vespa.ai"> + <img src="/img/vespa-logo.png" width="100" height="28"> + </a> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav navbar-right"> + <li><a href="http://blog.vespa.ai/">Blog</a> + <li><a href="https://twitter.com/vespaengine">Twitter</a> + <li><a href="/documentation/">Docs</a> + <li><a href="https://github.com/vespa-engine" target="_blank">GitHub</a> + <li><a href="https://github.com/vespa-engine/vespa/issues" target="_blank">Issues</a> + </ul> + <div class="col-sm-offset-2 col-md-offset-2"> + <div class="row"> + <form class="search" action="/search.html" method="get" id="search-form"> + <div class="col-xs-4"> + <input type="text" class="form-control" placeholder="Search Documentation" id="search" name="q"> + </div> + <button type="submit" id="submit" class="btn btn-search"><i class="fa fa-search" aria-hidden="true"></i></button> + </form> + </div> + </div> + </div> + </div> +</nav> + + +<div id="main-content" class="container-fluid"> + <div class="row"> + + <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + + <div class="col-sm-3 col-md-2 sidebar"> + <ul class="nav nav-sidebar"> + <li class="collapse-all" onclick="expand_all();">[+] expand all</li> + </ul> + + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">WELCOME</li> + + + + <li class="collapseable"><a href="/">Welcome to Vespa</a></li> + + + + + <li class="collapseable"><a href="/documentation/overview.html">Vespa Overview</a></li> + + + + + <li class="collapseable"><a href="/documentation/features.html">Features</a></li> + + + + + <li class="collapseable"><a href="/documentation/elastic-search-comparison.html">Comparison to Elasticsearch</a></li> + + + + + <li class="collapseable"><a href="/documentation/contributing.html">Contributing to Vespa</a></li> + + + + + <li class="collapseable"><a href="/documentation/introduction-to-documentation.html">Documentation Conventions</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">GETTING STARTED</li> + + + + <li class="collapseable"><a href="/documentation/vespa-quick-start.html">Quick Start</a></li> + + + + + <li class="collapseable"><a href="/documentation/tutorials/blog-search.html">Blog Search Tutorial</a></li> + + + + + <li class="collapseable"><a href="/documentation/build-vespa.html">Build Vespa</a></li> + + + + + <li class="collapseable"><a href="/documentation/api.html">Vespa API and Interfaces</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">DOCUMENTS AND SEARCH DEFINITIONS</li> + + + + <li class="collapseable"><a href="/documentation/documents.html">Documents</a></li> + + + + + <li class="collapseable"><a href="/documentation/search-definitions.html">Search Definitions</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">WRITING TO VESPA</li> + + + + <li class="collapseable"><a href="/documentation/writing-to-vespa.html">Writing to Vespa</a></li> + + + + + <li class="collapseable"><a href="/documentation/vespa-http-client.html">Vespa Feeding Client API</a></li> + + + + + <li class="collapseable"><a href="/documentation/feed-using-hadoop-pig-oozie.html">Feed using Hadoop, Pig, Oozie</a></li> + + + + + <li class="collapseable"><a href="/documentation/document-api-guide.html">Document API</a></li> + + + + + <li class="collapseable"><a href="/documentation/document-processing-overview.html">Document Processing</a></li> + + + + + <li class="collapseable"><a href="/documentation/annotations.html">Annotations API</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">QUERYING VESPA</li> + + + + <li class="collapseable"><a href="/documentation/querying-vespa.html">Querying Vespa</a></li> + + + + + <li class="collapseable"><a href="/documentation/search-api.html">Search API</a></li> + + + + + <li class="collapseable"><a href="/documentation/query-language.html">Vespa Query Language</a></li> + + + + + <li class="collapseable"><a href="/documentation/grouping.html">Grouping Information in Results</a></li> + + + + + <li class="collapseable"><a href="/documentation/federation.html">Federation</a></li> + + + + + <li class="collapseable"><a href="/documentation/query-profiles.html">Query Profiles</a></li> + + + + + <li class="collapseable"><a href="/documentation/geo-search.html">Geo Search</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">RANKING AND ML MODELS</li> + + + + <li class="collapseable"><a href="/documentation/ranking.html">Ranking Introduction</a></li> + + + + + <li class="collapseable"><a href="/documentation/nativerank.html">Ranking With nativeRank</a></li> + + + + + <li class="collapseable"><a href="/documentation/tensor-intro.html">Tensor Introduction</a></li> + + + + + <li class="collapseable"><a href="/documentation/tensor-user-guide.html">Tensor User Guide</a></li> + + + + + <li class="collapseable"><a href="/documentation/tensorflow.html">Ranking With TensorFlow Models</a></li> + + + + + <li class="collapseable"><a href="/documentation/onnx.html">Ranking With ONNX Models</a></li> + + + + + <li class="collapseable"><a href="/documentation/xgboost.html">Ranking With XGBoost Models</a></li> + + + + + <li class="collapseable"><a href="/documentation/stateless-model-evaluation.html">Stateless model evaluation</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">LINGUISTICS AND TEXT PROCESSING</li> + + + + <li class="collapseable"><a href="/documentation/linguistics.html">Linguistics in Vespa</a></li> + + + + + <li class="collapseable"><a href="/documentation/stemming.html">Stemming</a></li> + + + + + <li class="collapseable"><a href="/documentation/text-processing.html">Special Tokens</a></li> + + + + + <li class="collapseable"><a href="/documentation/query-rewriting.html">Query Rewriting</a></li> + + + + + <li class="collapseable"><a href="/documentation/query-phrasing.html">Query Phrasing With PhrasingSearcher</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">DEVELOPING APPLICATIONS AND PLUGINS</li> + + + + <li class="collapseable"><a href="/documentation/cloudconfig/application-packages.html">Application Packages</a></li> + + + + + <li class="collapseable"><a href="/documentation/container-intro.html">The Vespa Container</a></li> + + + + + <li class="collapseable"><a href="/documentation/jdisc/">Java Data Intensive Serving Container - JDisc</a></li> + + + + + <li class="collapseable"><a href="/documentation/jdisc/container-components.html">JDisc Container Components</a></li> + + + + + <li class="collapseable"><a href="/documentation/component-types.html">Container Component Types</a></li> + + + + + <li class="collapseable"><a href="/documentation/jdisc/developing-applications.html">Application Development Basics</a></li> + + + + + <li class="collapseable"><a href="/documentation/jdisc/processing.html">Request-Response Processing</a></li> + + + + + <li class="collapseable"><a href="/documentation/searcher-development.html">Searcher Development</a></li> + + + + + <li class="collapseable"><a href="/documentation/docproc-development.html">Document Processor Development</a></li> + + + + + <li class="collapseable"><a href="/documentation/developing-web-services.html">Developing Web Service Applications</a></li> + + + + + <li class="collapseable"><a href="/documentation/handler-tutorial.html">HTTP API Use Case Tutorial</a></li> + + + + + <li class="collapseable"><a href="/documentation/jdisc/injecting-components.html">Component Injection</a></li> + + + + + <li class="collapseable"><a href="/documentation/chained-components.html">Chained Components</a></li> + + + + + <li class="collapseable"><a href="/documentation/jdisc/developing-osgi-bundles.html">Building OSGi Bundles</a></li> + + + + + <li class="collapseable"><a href="/documentation/bundle-plugin.html">OSGi Bundle Maven Plugin</a></li> + + + + + <li class="collapseable"><a href="/documentation/jdisc/component-versioning.html">Versioning in the Container</a></li> + + + + + <li class="collapseable"><a href="/documentation/jdisc/metrics.html">Container Metrics</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">CONFIGURATION</li> + + + + <li class="collapseable"><a href="/documentation/cloudconfig/config-introduction.html">The Cloud Configuration System</a></li> + + + + + <li class="collapseable"><a href="/documentation/cloudconfig/configapi-dev.html">Cloud Config API</a></li> + + + + + <li class="collapseable"><a href="/documentation/cloudconfig/cloudconfig-model-plugins.html">Developing Cloud Config Model Plugins</a></li> + + + + + <li class="collapseable"><a href="/documentation/cloudconfig/deploy-rest-api-v2.html">Deploy API</a></li> + + + + + <li class="collapseable"><a href="/documentation/cloudconfig/config-rest-api-v2.html">Config API</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">RESULT FORMATS</li> + + + + <li class="collapseable"><a href="/documentation/document-summaries.html">Document Summaries</a></li> + + + + + <li class="collapseable"><a href="/documentation/result-rendering.html">Search Result Renderers</a></li> + + + + + <li class="collapseable"><a href="/documentation/page-templates.html">Page Templates</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">PREDICATE SEARCH</li> + + + + <li class="collapseable"><a href="/documentation/predicate-fields.html">Predicate Fields</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">USER AND GROUP ORIENTED SEARCH</li> + + + + <li class="collapseable"><a href="/documentation/streaming-search.html">Streaming Search</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">THE CONTENT LAYER</li> + + + + <li class="collapseable"><a href="/documentation/elastic-vespa.html">Elastic Vespa</a></li> + + + + + <li class="collapseable"><a href="/documentation/proton.html">Proton</a></li> + + + + + <li class="collapseable"><a href="/documentation/content/consistency.html">Vespa Consistency Model</a></li> + + + + + <li class="collapseable"><a href="/documentation/content/idealstate.html">Distribution Algorithm</a></li> + + + + + <li class="collapseable"><a href="/documentation/content/data-placement.html">Document Distribution</a></li> + + + + + <li class="collapseable"><a href="/documentation/content/buckets.html">Buckets</a></li> + + + + + <li class="collapseable"><a href="/documentation/content/admin-states.html">Cluster and Node States</a></li> + + + + + <li class="collapseable"><a href="/documentation/content/api-state-rest-api.html">State API</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">PERFORMANCE AND TUNING</li> + + + + <li class="collapseable"><a href="/documentation/performance/sizing-search.html">Vespa Search Sizing Guide</a></li> + + + + + <li class="collapseable"><a href="/documentation/attributes.html">Document Attributes</a></li> + + + + + <li class="collapseable"><a href="/documentation/performance/attribute-memory-usage.html">Attribute Memory Usage</a></li> + + + + + <li class="collapseable"><a href="/documentation/performance/vespa-benchmarking.html">Vespa Benchmarking</a></li> + + + + + <li class="collapseable"><a href="/documentation/performance/profiling-search-container.html">Profiling the Search Container</a></li> + + + + + <li class="collapseable"><a href="/documentation/performance/container-tuning.html">Container Tuning</a></li> + + + + + <li class="collapseable"><a href="/documentation/performance/rate-limiting-searcher.html">Rate-Limiting Search Requests</a></li> + + + + + <li class="collapseable"><a href="/documentation/performance/fbench.html">Vespa HTTP Benchmark Tool - vespa-fbench</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">OPERATIONS AND PROCEDURES</li> + + + + <li class="collapseable"><a href="/documentation/operations/admin-procedures.html">Administrative Procedures</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/vespa-cmdline-tools.html">Vespa Command-line Tools</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/metrics-health-format.html">Gathering Metrics from Vespa</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/logs.html">Log Files</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/files-processes-and-ports.html">Files, Processes and Ports</a></li> + + + + + <li class="collapseable"><a href="/documentation/inspecting-java-services.html">Inspecting Vespa Java Services</a></li> + + + + + <li class="collapseable"><a href="/documentation/docker-containers-in-production.html">Docker Containers in Production</a></li> + + + + + <li class="collapseable"><a href="/documentation/securing-your-vespa-installation.html">Securing a Vespa installation</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">MIGRATING FROM ELASTICSEARCH TO VESPA</li> + + + + <li class="collapseable"><a href="/documentation/migrating-from-elastic-search-to-vespa.html">Migrating from Elasticsearch to Vespa</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">CONFIGURATION REFERENCE</li> + + + + <li class="collapseable"><a href="/documentation/reference/application-packages-reference.html">Application Package Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/search-definitions-reference.html">Search Definition Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/services.html">Services Configuration (services.xml)</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/advanced-indexing-language.html">Indexing Language</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/config-files.html">Custom Configuration File Reference</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">RANKING REFERENCE</li> + + + + <li class="collapseable"><a href="/documentation/reference/ranking-expressions.html">Ranking Expressions</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/tensor.html">Tensor Evaluation Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/nativerank.html">nativeRank Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/rank-features.html">Rank Feature Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/string-segment-match.html">String Segment Match</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/rank-feature-configuration.html">Rank Feature Configuration</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/rank-types.html">Rank Types</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">QUERIES AND RESULTS REFERENCE</li> + + + + <li class="active collapseable"><a href="/documentation/reference/search-api-reference.html">Search API Reference <span class="sr-only">(current)</span></a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/simple-query-language-reference.html">Simple Query Language Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/select-reference.html">Select Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/grouping-syntax.html">Grouping Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/sorting.html">Sorting Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/query-profile-reference.html">Query Profile Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/semantic-rules.html">Semantic Rule Language Reference</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/advanced-search-operators.html">Advanced Search Operators</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/equiv.html">The EQUIV Operator</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/wand-operator.html">The WAND Operator</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/weighted-set-term.html">The WeightedSetItem Operator</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/dot-product-search-operator.html">The Dot Product Search Operator</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/default-result-format.html">Default JSON Result Format</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/page-templates-syntax.html">Page Templates Syntax</a></li> + + + + + </ul> + + <ul class="nav nav-sidebar"> + <li class="collapse-parent" onclick="on_collapse(this);">DOCUMENT FORMAT AND LANGUAGES REFERENCE</li> + + + + <li class="collapseable"><a href="/documentation/reference/document-json-format.html">Document JSON Format</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/document-field-path.html">Document Field Path Syntax</a></li> + + + + + <li class="collapseable"><a href="/documentation/reference/document-select-language.html">Document Selector Language</a></li> + + + + + </ul> + + + </div> + + <script type="application/javascript"> + + $(document).ready (function() { + $(".collapseable").each(function() { + $(this).addClass("collapsed"); + }); + $(".active").each(function() { + $(this).removeClass("collapsed"); + $(this).siblings().each(function() { + if (!$(this).hasClass("collapse-parent")) { + $(this).removeClass("collapsed"); + } + }); + }); + }); + + function on_collapse(e) { + $(e).siblings().each(function() { + $(this).toggleClass("collapsed"); + }); + } + + function expand_all() { + $(".collapseable").each(function() { + $(this).removeClass("collapsed"); + }); + } + +</script> + + + <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main"> + + <h1> + Search API Reference + <div class="pull-right"><a href="https://github.com/vespa-engine/documentation/blob/master/documentation/reference/search-api-reference.html" class="btn btn-link"><i class="fa fa-github"></i> edit page</a></div> + </h1> + + <p> + All the search request parameters listed below can be set in query + profiles. The first four blocks of properties are also modeled as + query profile types. These types can be referred from query profiles + (and inheriting types) to provide type checking on the parameters. + </p><p> + These parameters often have both a full name - which includes the + path from the root query profile - and one or more abbreviated + names. Both names can be used in search requests, while only full + names can be used in query profiles. The full names are case + sensitive, while the abbreviated names are case insensitive. + </p><p> + The parameters modeled as query profiles are also available through + get methods as Java objects from the Query to Searcher components. + </p> + + + + <h2>Index</h2> + + <dt>Query</dt> + <dd> + <ul> + <li><a href="#yql">yql</a></li> + <li><a href="#Select">select</a></li> + <ul> + <li><a href="#where">where</a></li> + <li><a href="#grouping">grouping</a></li> + </ul> + </ul> + </dd> + </dl> + + <dl> + <dt>Native Execution Parameters</dt> + <dd> + <ul> + <li><a href="#hits">hits</a> [<em>count</em>]</li> + <li><a href="#offset">offset </a>[<em>start</em>]</li> + <li><a href="#queryProfile">queryProfile</a></li> + <li><a href="#nocache">nocache</a></li> + <li><a href="#groupingSessionCache">groupingSessionCache</a></li> + <li><a href="#searchChain">searchChain</a></li> + <li><a href="#timeout">timeout</a></li> + <li><a href="#tracelevel">tracelevel</a></li> + <li><a href="#trace.timestamps">trace.timestamps</a></li> + </ul> + </dd> + + <dt>Query Model Parameters</dt> + <dd> + <ul> + <li><a href="#model.defaultIndex">model.defaultIndex</a> [<em>default-index</em>]</li> + <li><a href="#model.encoding">model.encoding</a> [<em>encoding</em>]</li> + <li><a href="#model.filter">model.filter</a> [<em>filter</em>]</li> + <li><a href="#model.language">model.language</a> [<em>lang, language</em>]</li> + <li><a href="#model.queryString">model.queryString</a> [<em>query</em>]</li> + <li><a href="#model.restrict">model.restrict</a> [<em>restrict</em>]</li> + <li><a href="#model.searchPath">model.searchPath</a> [<em>path</em>]</li> + <li><a href="#model.sources">model.sources</a> [<em>search, sources</em>]</li> + <li><a href="#model.type">model.type</a> [<em>type</em>]</li> + </ul> + </dd> + + <dt>Ranking</dt> + <dd> + <ul> + <li><a href="#ranking.location">ranking.location</a> [<em>location</em>]</li> + <li><a href="#ranking.features">ranking.features</a> [<em>rankfeature</em>]</li> + <li><a href="#ranking.listFeatures">ranking.listFeatures</a> [<em>rankfeatures</em>]</li> + <li><a href="#ranking.profile">ranking.profile</a> [<em>ranking</em>]</li> + <li><a href="#ranking.properties">ranking.properties</a> [<em>rankproperty</em>]</li> + <li><a href="#ranking.sorting">ranking.sorting</a> [<em>sorting</em>]</li> + <li><a href="#ranking.freshness">ranking.freshness</a></li> + <li><a href="#ranking.queryCache">ranking.queryCache</a></li> + <li><a href="#ranking.matchPhase">ranking.matchPhase</a></li> + </ul> + </dd> + + <dt>Presentation</dt> + <dd> + <ul> + <li><a href="#presentation.bolding">presentation.bolding</a> [<em>bolding</em>]</li> + <li><a href="#presentation.format">presentation.format</a> [<em>format</em>]</li> + <li><a href="#presentation.template">presentation.template</a></li> + <li><a href="#presentation.summary">presentation.summary</a> [<em>summary</em>]</li> + <li><a href="#presentation.timing">presentation.timing</a></li> + </ul> + </dd> + + <dt>Grouping</dt> + <dd> + <ul> + <li><a href="#select">select</a></li> + <li><a href="#collapse.summary">collapse.summary</a></li> + <li><a href="#collapsefield">collapsefield</a></li> + <li><a href="#collapsesize">collapsesize</a></li> + </ul> + </dd> + + <dt>Geographical Searches</dt> + <dd> + <ul> + <li><a href="#pos.ll">pos.ll</a></li> + <li><a href="#pos.radius">pos.radius</a>,</li> + <li><a href="#pos.attribute">pos.attribute</a></li> + <li><a href="#pos.bb">pos.bb</a></li> + </ul> + </dd> + + <dt>Streaming Search</dt> + <dd> + <ul> + <li><a href="#streaming.userid">streaming.userid</a></li> + <li><a href="#streaming.groupname">streaming.groupname</a></li> + <li><a href="#streaming.selection">streaming.selection</a></li> + <li><a href="#streaming.priority">streaming.priority</a></li> + <li><a href="#streaming.maxbucketspervisitor">streaming.maxbucketspervisitor</a></li> + </ul> + </dd> + + <dt>Semantic Rules</dt> + <dd> + <ul> + <li><a href="#rules.off">rules.off</a></li> + <li><a href="#rules.rulebase">rules.rulebase</a></li> + <li><a href="#tracelevel.rules">tracelevel.rules</a></li> + </ul> + </dd> + + <dt>Other</dt> + <dd> + <ul> + <li><a href="#recall">recall</a></li> + <li><a href="#user">user</a></li> + <li><a href="#nocachewrite">nocachewrite</a></li> + <li><a href="#hitcountestimate">hitcountestimate</a></li> + <li><a href="#metrics.ignore">metrics.ignore</a></li> + </ul> + </dd> + </dl> + + + <h2 id="query">Query</h2> + <h3 id="yql">yql</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>String</td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + <p> + The YQL query will be parsed and executed in the backend. + Only simple YQL programs are supported, refer to + <a href="../query-language.html">YQL</a> for details. + </p> + + <h3 id="Select">select</h3> + <p>Select query is equivalent with YQL, written in JSON. Contains subparameters <code>where</code> and <code>grouping</code>.</p> + + <h4 id="where"> where</h4> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>JSON</td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + + <h4 id="grouping"> grouping</h4> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>JSON</td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + <p> + The where and grouping query will be parsed and executed in the backend. + Refer to + <a href="../reference/select-reference.html">Select Reference</a> for details. + </p> + + + + + <h2 id="native-execution-parameters">Native Execution Parameters</h2> + <p> + These parameters are defined in the <code>native</code> query profile type. + </p> + + + <h3 id="hits">hits</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>count</td></tr> + <tr><td>Values</td> + <td> + A positive integer, or 0. The sum of <a href="#offset">offset</a> and + <a href="#hits">hits</a> should be lower than the configured maxoffset + value, and will be adjusted to fit. See also comment + at <code>offset</code>. + </td> + </tr> + <tr><td>Default</td><td>10</td></tr> + </table> + <p> + The maximum number of hits to return from the result set. + Must be lower than <code>maxHits</code>, which is either set in a + <a href="#queryProfile">query profile</a>, or default 400. + <!-- ToDo: link to def file or code where this is definied --> + </p> + + + <h3 id="offset">offset</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>start</td></tr> + <tr><td>Values</td> + <td> + A positive integer, including 0. + </td> + </tr> + <tr><td>Default</td><td>0</td></tr> + </table> + <p> + The index of the first hit to return from the result set. + Must be lower than <code>maxOffset</code>, which is either set in a + <a href="#queryProfile">query profile</a>, or default 1000. + <!-- ToDo: link to def file or code where this is definied --> + </p> + + + <h3 id="queryProfile">queryProfile</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td><em>None</em></td></tr> + <tr><td>Values</td> + <td> + A query profile id - name:version, where version can be omitted + or partially specified, e.g "myprofile:2.1" + </td> + </tr> + <tr><td>Default</td><td><code>default</code></td></tr> + </table> + <p> + A <a href="../query-profiles.html">query profile</a> has default properties for a query. + The default query profile is named <em>default</em> - example: + <pre> +<query-profile id="default"> + <field name="maxHits">10</field> + <field name="maxOffset">1000</field> +</query-profile> +</pre> + </p> + + + <h3 id="nocache">nocache</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td> + <td> + True or false + </td> + </tr> + <tr><td>Default</td><td>false</td></tr> + </table> + <p> + Set to true to avoid the result being fetched from cache, and avoid + writing the result to cache after fetching it. + </p> + + + <h3 id="groupingSessionCache">groupingSessionCache</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td> + <td> + True or false + </td> + </tr> + <tr><td>Default</td><td>false</td></tr> + </table> + <p> + Set to true to store intermediate grouping results in the search back ends when + using multi level grouping expressions in order to speed up grouping at a + potential loss of accuracy. See the <a + href="grouping-syntax.html#sessionCache">grouping reference</a> for more + details. + </p> + + + <h3 id="searchChain">searchChain</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td> + <td> + A search chain id - name:version, where version can be + omitted or partially specified, e.g "mychain:2.1.3". + </td> + </tr> + <tr><td>Default</td><td><code>default</code></td></tr> + </table> + <p> + The search chain initially invoked when processing this query. This + search chain may invoke other chains. + </p> + + + <h3 id="timeout">timeout</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td> + <td> + Positive floating point number with an optional unit. Default unit + is seconds (s), valid unit strings are e.g. <em>ms</em> and <em>s</em>. To set + a timeout of one minute, the argument could be set to <em>60 s</em>. + Space between the number and the unit is optional. + </td> + </tr> + <tr><td>Default</td><td>Undefined, but guaranteed to be at least 5000 milliseconds. This default can be overridden by configuring timeout in a <a href="../query-profiles.html">query profile.</a></td></tr> + </table> + <p>The query timeout.</p> + + + <h3 id="tracelevel">tracelevel</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td> + <td> + Any positive number + </td> + </tr> + <tr><td>Default</td><td><em>No tracing</em></td></tr> + </table> + <p> + Set to a positive number to collect trace information for debugging + when running a query. Higher numbers give + progressively more detail on query transformations and searcher + execution. + </p> + + <h3 id="trace.timestamps">trace.timestamps</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td> + <td> + true or false + </td> + </tr> + <tr><td>Default</td><td><em>No timestamps in trace</em></td></tr> + </table> + <p> + Enable it to get timing information already at <a href="#tracelevel">tracelevel=1</a> which is useful for debugging latency spent at different components in the search chain without rendering a lot of string data which is associated with higher trace levels. + </p> + + + + <h2 id="query-model">Query Model Parameters</h2> + + <h3 id="model.defaultIndex">model.defaultIndex [<em>default-index</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>default-index</td></tr> + <tr><td>Values</td><td>An index name</td></tr> + <tr><td>Default</td><td><code>default</code></td></tr> + </table> + <p> + The field which is searched for query terms which doesn't explicitly specify an index. + </p> + + + <h3 id="model.encoding">model.encoding [<em>encoding</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>encoding</td></tr> + <tr><td>Values</td><td>Encoding names or aliases defined in the <a href="http://www.iana.org/assignments/character-sets">IANA character sets</a></td></tr> + <tr><td>Default</td><td>utf-8</td></tr> + </table> + <p> Sets the encoding to use when returning a result. The encodings <em>big5</em>, + <em>euc-jp</em>, <em>euc-kr</em>, <em>gb2312</em>, <em>iso-2022-jp</em> and <em>shift-jis</em> + also influences how tokenization is done in the absence of an explicit language setting. + </p><p> + The query is always encoded as UTF-8, independently of how the result will be encoded. + </p> + + + <h3 id="model.filter">model.filter [<em>filter</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>filter</td></tr> + <tr><td>Values</td><td>Any allowed collection of filter terms</td></tr> + <tr><td>Default</td><td><em>Not set</em></td></tr> + </table> + <p> + Sets a filter to be combined with the query. Typical use of a filter + is to add machine generated or preferences based filter terms to a raw + user query. The filter is parsed the same way as a query of type any, + the full syntax is available. The positive terms (preceded by +) and + phrases act as AND filters, the negative terms (preceded by -) act as + NOT filters, while the unprefixed terms will be used to RANK the + results. Unless the query has no positive terms, the filter will only + restrict and influence ranking of the result set, never cause more + matches than the query. + </p> + + + <h3 id="model.language">model.language [<em>lang, language</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>language, lang</td></tr> + <tr><td>Values</td><td>Ref. RFC 3066</td></tr> + <tr><td>Default</td><td></em>Unspecified</em></td></tr> + </table> + <p> + Informs Vespa about the natural language of the query. Please see + <a href="../linguistics.html">linguistics</a> for details. + This attribute should always be set when it is known. If this + parameter is not set, it will be guessed from the query and encoding, and + default to english if it cannot be guessed. + </p> + + + <h3 id="model.queryString">model.queryString [<em>query</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>query</td></tr> + <tr><td>Values</td><td>Any HTTP encoded legal Vespa query language string</td></tr> + <tr><td>Default</td><td><em>Not set</em></td></tr> + </table> + <p> + The <a href="simple-query-language-reference.html">Simple Vespa Query Language</a> query string + specifying which documents to match in this query. + </p> + + + <h3 id="model.restrict">model.restrict [<em>restrict</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>restrict</td></tr> + <tr><td>Values</td><td>A comma delimited list of document type names.</td></tr> + <tr><td>Default</td><td><em>Search unrestricted</em></td></tr> + </table> + <p> + The document types to restrict the search to when different document + types share the same search cluster. + </p> + + + <h3 id="model.searchPath">model.searchPath [<em>path</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>searchpath</td></tr> + <tr><td>Values</td><td><ul> + <li>searchpath::ELEMENT [';' ELEMENT]*</li> + <li>ELEMENT::PART ['/' ROW]</li> + <li>PART::EXP [',' EXP]*</li> + <li>EXP::NUM | RANGE</li> + <li>ROW::NUM</li> + <li>RANGE::'['NUM ',' NUM ' >'</li> + </ul></td></tr> + <tr><td>Default</td><td><em>Whole cluster</em></td></tr> + </table> + <p> + Specification of which path to send the query to. + Used to select which set of search nodes in the cluster should be used. + Only meant for debugging/monitoring. + </p><p> + Examples: + Note that in an indexed content cluster with flat distribution we have 1 implicit row + and each search node represents a part. + <ul> + <li>'7/3' = part 7, row 3.</li> + <li>'7/' = part 7, any row.</li> + <li>'7,1,9/0' = parts 1,7 and 9, row 0.</li> + <li>'1,[3,9>/0' = parts 1,3,4,5,6,7,8, row 0.</li> + </ul> + </p><p> + In a cluster with a multi-level dispatch setup we must specify a search path element for each level. + Lets say we have a setup with 2 mid-level dispatch groups, each containing 3 search nodes (and 3 dispatchers): + <ul> + <li>'0/;2/' = dispatch group (part) 0, any of the dispatchers (row); search node (part) 2, any row (of 1 present).</li> + <li>'0/1;2/0' = dispatch group (part) 0, dispatcher (row) 1; search node (part) 2, row 0 (of 1 present).</li> + </ul> + </p> + + + <h3 id="model.sources">model.sources [<em>search, sources</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>search, sources</td></tr> + <tr><td>Values</td><td>A comma separated list of search cluster names or other source names</td></tr> + <tr><td>Default</td><td><em>Search unrestricted</em></td></tr> + </table> + <p> + The names of the sources to search, e.g one or more search clusters and/or federated sources. + </p> + + + <h3 id="model.type">model.type [<em>type</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>type</td></tr> + <tr><td>Values</td><td>web, all, any, phrase, yql, adv (deprecated) - + refer to <a href="simple-query-language-reference.html">simple query language reference</a></td></tr> + <tr><td>Default</td><td>all</td></tr> + </table> + <p> + Selects the query language syntax of the <a href="#model.queryString">query</a> parameter. + </p> + + + + <h2 id="ranking">Ranking</h2> + + <h3 id="ranking.location">ranking.location [<em>location</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>location</td></tr> + <tr><td>Values</td><td>See <a href="../geo-search.html">Geo search</a></td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + <p> + Point (one or two dimensional) location to use as base for location ranking. + For geographical locations, it is recommended to add the location using <a href="#pos.ll">pos.ll</a> + <!-- ToDo: Why? --> + </p> + + + <h3 id="ranking.features">ranking.features.<em>featurename</em> [<em>rankfeature.</em>featurename]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>rankfeature.featurename</td></tr> + <tr><td>Values</td><td>Any string</td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + <p> + Set a rank feature to a value. This works for any key name <code>query(anyname)</code> (query features), + and also as a way to override all existing (match and document) features. + Example: <em>query=foo&ranking.features.query(userage)=42&ranking.features.fieldMatch(title)=0.65</em> + </p> + + + <h3 id="ranking.listFeatures">ranking.listFeatures [<em>rankfeatures</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>rankfeatures</td></tr> + <tr><td>Values</td><td>boolean</td></tr> + <tr><td>Default</td><td>false</td></tr> + </table> + <p> + Set to true to request <em>all</em> rank features to be calculated and returned. + The rank features will be returned in the summary field rankfeatures. + This option is typically used for MLR training, should not to be used for production. + </p> + + + <h3 id="ranking.profile">ranking.profile [<em>ranking</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>ranking</td></tr> + <tr><td>Values</td><td>Any rank profile name</td></tr> + <tr><td>Default</td><td><code>default</code></td></tr> + </table> + <p> + Sets the name of the rank profile to use for assigning relevancy scores. + The default rank profile will be used for back-ends which does not have the given rank profile. + </p> + + + <h3 id="ranking.properties">ranking.properties.<em>propertyname</em> [<em>rankproperty.propertyname</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>rankproperty.propertyname</td></tr> + <tr><td>Values</td><td>Any string</td></tr> + <tr><td>Default</td><td><em>None</em></td></tr> + </table> + <p> + Set a rank property that is passed to, and used by a feature executor for this query. + Example: <em>query=foo&ranking.properties.dotProduct.X={a:1,b:2}</em> + </p> + + + <h3 id="ranking.sorting">ranking.sorting [<em>sorting</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>sorting</td></tr> + <tr><td>Values</td><td>A valid <a href="sorting.html">sort specification</a></td></tr> + <tr><td>Default</td><td>None - order by relevance</td></tr> + </table> + <p> + A specification of how to sort the result. + Fields you want to sort on must be stored as document attributes in the index structure + by adding <a href="search-definitions-reference.html#attribute">attribute</a> to the indexing statement. + </p> + + + <h3 id="ranking.freshness">ranking.freshness</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td><code>[integer]</code>, an absolute time in seconds since epoch, or <code>now-[number]</code>, to use a time [integer] seconds into the past, or <code>now</code> to use the current time</td></tr> + <tr><td>Default</td><td>None - use the current time on each node.</td></tr> + </table> + <p> + Sets the time which will be used as <em>now</em> during execution. + </p> + + + <h3 id="ranking.queryCache">ranking.queryCache</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>boolean</td></tr> + <tr><td>Default</td><td>false</td></tr> + </table> + <p> + Turns query cache on or off. Search is a two-phase process. If the + query cache is on, the query is stored on the search nodes between the + first and second phase, saving network bandwidth and also query setup + time, at the expense of using more memory. + </p> + + + <h3 id="ranking.matchPhase">ranking.matchPhase</h3> + <p>Settings which control Vespa's behavior during the match phase. + If these are set in the query they will override any match-phase setting + in the rank profile.</p> + <dt></dt> + <dd> + <ul> + <li><a href="#ranking.matchPhase.maxHits">ranking.matchPhase.maxHits</a> the max number of hits that should be generated during the match phase</li> + <li><a href="#ranking.matchPhase.attribute">ranking.matchPhase.attribute</a> the attribute to limit matches by if more than maxHits hits will be generated</li> + <li><a href="#ranking.matchPhase.ascending">ranking.matchPhase.ascending</a> whether to keep the documents having the highest (default) or lowest values of the attribute</li> + <li><a href="#ranking.matchPhase.diversity.attribute">ranking.matchPhase.diversity.attribute</a> the attribute to use to guarantee diversity.</li> + <li><a href="#ranking.matchPhase.diversity.minGroups">ranking.matchPhase.diversity.minGroups</a> the minimum number of groups grouped by the diversity attribute.</li> + </ul> + </dd> + + + <h3 id="ranking.matchPhase.maxHits">ranking.matchPhase.maxHits</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>long</td></tr> + <tr><td>Default</td><td>If sorting and not ranking: max(10000, maxhits+maxoffset). + Otherwise: <em>none</em>.</td></tr> + </table> + <p> + The max hits the engine should attempt to produce in the match phase on each partition. + If it is determined during matching that many more hits than this will be generated, the matching will fall back to + take the best (highest or lowest) values of the attribute given by ranking.matchPhase.attribute. + </p><p> + By default, this will be turned on only when sorting is used and grouping is not. + If sorting is used, the primary sort attribute will be used as the match phase attribute if it has fast-search set. + In that case the default can be overridden by setting this value explicitly. + </p> + + + <h3 id="ranking.matchPhase.attribute">ranking.matchPhase.attribute</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>An attribute name</td></tr> + <tr><td>Default</td><td><em>none</em></td></tr> + </table> + <p> + The attribute to decide which documents are a match if the match phase + estimates that there will be more than maxHits matches. + This attribute should have fast-search set and should correlate with the order + which would be produced by a full evaluation. + </p> + + + <h3 id="ranking.matchPhase.ascending">ranking.matchPhase.ascending</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>boolean</td></tr> + <tr><td>Default</td><td>false</td></tr> + </table> + <p> + Whether the attribute should be sorted in ascending or descending (default) order + to determine which documents to keep as matches. + </p> + + + <h3 id="ranking.matchPhase.diversity.attribute">ranking.matchPhase.diversity.attribute</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>An attribute name</td></tr> + <tr><td>Default</td><td>none.</td></tr> + </table> + <p> + The attribute to be used for producing the desired diversity. + Also see <a href="search-definitions-reference.html#diversity-attribute">attribute</a>. + </p> + + + <h3 id="ranking.matchPhase.diversity.minGroups">ranking.matchPhase.diversity.minGroups</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>long</td></tr> + <tr><td>Default</td><td>none</td></tr> + </table> + <p> + The minimum number of groups that should be returned from the match phase grouped by the diversity attribute. + Also see <a href="search-definitions-reference.html#diversity-min-groups">min-groups</a>. + </p> + + + + <h2 id="presentation">Presentation</h2> + + <h3 id="presentation.bolding">presentation.bolding [<em>bolding</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>bolding</td></tr> + <tr><td>Values</td><td>boolean</td></tr> + <tr><td>Default</td><td>true</td></tr> + </table> + <p> + Whether or not to bold search terms in <a href="search-definitions-reference.html">search definition</a> + fields defined with <a href="search-definitions-reference.html#bolding">bolding: on</a> + or <a href="search-definitions-reference.html#summary">summary: dynamic</a>. + </p> + + + <h3 id="presentation.format">presentation.format [<em>format</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>format</td></tr> + <tr><td>Values</td><td> + <table class="table table-striped"> + <tr> + <td><em>No value</em> or <code><a href="default-result-format.html">default</a></code></td> + <td>The default, builtin JSON format</td> + </tr> + <tr> + <td><code><a href="default-result-format.html">json</a></code></td> + <td>Builtin JSON format</td> + </tr> + <tr> + <td><code>xml</code></td> + <td>Deprecated, builtin XML format</td> + </tr> + <tr> + <td><code><a href="page-result-format.html">page</a></code></td> + <td>Alternative deprecated XML format which is suitable for use with <a href="../page-templates.html">page templates</a>.</td> + </tr> + <tr> + <td><em>Any other value</em></td> + <td>A custom <a href="../result-rendering.html">result renderer</a> supplied by the application + </tr> + </table> + + </td></tr> + <tr><td>Default</td><td>default</td></tr> + </table> + + + <h3 id="presentation.summary">presentation.summary [<em>summary</em>]</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td>summary</td></tr> + <tr><td>Values</td><td> + The name of the <a href="../document-summaries.html#summary-classes-in-queries">summary class</a> + used to select fields in results. + </td></tr> + <tr><td>Default</td><td>The default summary class of the search definition.</td></tr> + </table> + + + <h3 id="presentation.template">presentation.template</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>Any id specification of a deployed page template.</td></tr> + <tr><td>Default</td><td></td></tr> + </table> + <p> + The id of the page template to use for this result. This should be used with the + <a href="page-result-format.html">page</a> result format. + </p> + + + <h3 id="presentation.timing">presentation.timing</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>boolean</td></tr> + <tr><td>Default</td><td>false</td></tr> + </table> + <p> + Whether a result renderer should try to add optional timing information + to the rendered page. + </p> + + + + <h2 id="">Grouping and Aggregation</h2> + + <h3 id="select">select</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>A valid grouping specification.</td></tr> + <tr><td>Default</td><td>No grouping</td></tr> + </table> + <p> + Requests specific multi-level result set statistics and/or hit groups to be returned in the result. + Fields you want to retrieve statistics or hit groups for must be stored as document attributes + in the index structure by adding attribute to the indexing statement. + See the <a href="../grouping.html">grouping guide</a>. + </p> + + + <h3 id="collapsefield">collapsefield</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>Any document summary field name</td></tr> + <tr><td>Default</td><td>No field collapsing</td></tr> + </table> + <p> + Collapse (i.e. aggregate) results using this field. + Collapsing is run in the container, not content node level. + Define a <em>collapsefield</em> to remove duplicates if the corpus has few duplicates - + this is more efficient than using <a href="#select">grouping</a>. + Otherwise, use grouping. + </p> + + + <h3 id="collapsesize">collapsesize</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>A positive integer</td></tr> + <tr><td>Default</td><td>1</td></tr> + </table> + <p>The number of hits to keep in each collapsed bucket</p> + + + <h3 id="collapse.summary">collapse.summary</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>A valid name of a document summary class.</td></tr> + <tr><td>Default</td><td>Use default summary or attributes.</td></tr> + </table> + <p>Use this summary class to fetch the field used for collapsing.</p> + + + + <h2 id="geographical-searches">Geographical Searches</h2> + + <h3 id="pos.ll">pos.ll</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td> + <td>Position given in latitude and longitude - example: <em>S22.4532;W123.9887</em> + Refer to <a href="search-definitions-reference.html#type:position">position field</a> + for format specification.</td> + </tr> + <tr><td>Default</td><td>None</td></tr> + </table> + + + <h3 id="pos.radius">pos.radius</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td> + Radius of the circle used for filtering. Valid units of measurement are km, m and mi. Examples: + <ul> + <li>pos.radius=100m</li> + <li>pos.radius=42mi</li> + <li>pos.radius=4km</li> + </ul> + One can also specify just a number (internal units, micro-degrees), but this is not recommended. + </td></tr> + <tr><td>Default</td><td>50km</td></tr> + </table> + + + <h3 id="pos.bb">pos.bb</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td> + Bounding box for positions, given as latitude and longitude boundaries. + The four boundaries must be specified as N, S, E, W, with degrees as + a decimal fraction. Degrees south of equator or west of Greenwich are + input as negative numbers. Examples: + <ul> + <li>n=37.44899,s=37.3323,e=-121.98241,w=-122.06566</li> + <li>s=40.183868,w=-74.819519,n=40.248291,e=-74.728798</li> + </ul> + </td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + + + <h3 id="pos.attribute">pos.attribute</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>Any attribute that has zcurve encoded positions as a long attribute.</td></tr> + <tr><td>Default</td><td>Random choice among the ones declared as position in the searchdefinition.</td></tr> + </table> + <p> + Which attribute to use for the position. Can be both single- or multi-value. + </p> + + + + <h2 id="">Streaming Search</h2> + <p> + The features in this section applies to <a href="../streaming-search.html">streaming search</a> only. + </p> + + <h3 id="streaming.userid">streaming.userid</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>An integer in decimal notation in the range [0, 2^64></td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + <p> + Restricts streaming search to only stream through documents with document ids having the n=<number> + modifier and the userid part matches the supplied value. This can be used for grouping documents on a 64 bit integer. + </p> + + + <h3 id="streaming.groupname">streaming.groupname</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>A string</td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + <p> + Restricts streaming search to only stream through documents with document ids having the g=<groupname> + modifier and the groupname part matches the supplied value. This can be used for grouping documents on a string. + </p> + + + <h3 id="streaming.selection">streaming.selection</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>A string</td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + <p> + Restricts streaming search using a <a href="document-select-language.html">document selection</a>. + This can be used for selecting a subset of documents based on an advanced expression. + </p> + + + <h3 id="streaming.priority">streaming.priority</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td><a href="services.html#load-types">Priority class</a></td></tr> + <tr><td>Default</td><td>VERY_HIGH</td></tr> + </table> + <p> + Priority of the streaming search visitor. Having a high priority visitor helps maintain low latencies + even when the system is under load. + </p> + + + <h3 id="streaming.maxbucketspervisitor">streaming.maxbucketspervisitor</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>int</td></tr> + <tr><td>Default</td><td>1 (if ordering is set), or infinite</td></tr> + </table> + <p> + If set, visit only this many buckets at a time. + Combine with ordering to reduce visiting time for large users/groups. + </p> + + + + <h2 id="semantic-rules">Semantic Rules</h2> + <p> + Refer to <a href="semantic-rules.html">semantic rules</a>. + </p> + + <h3 id="rules.off">rules.off</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>Boolean</td></tr> + <tr><td>Default</td><td>True</td></tr> + </table> + <p>Turn rule evaluation off for this query</p> + + + <h3 id="rules.rulebase">rules.rulebase</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>String</td></tr> + <tr><td>Default</td><td>A rule base name</td></tr> + </table> + <p>The name of the rule base to use for these queries</p> + + + <h3 id="tracelevel.rules">tracelevel.rules</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>int</td></tr> + <tr><td>Default</td><td>1-5 (?)</td></tr> + </table> + <p> + The amount of rule evaluation trace output to show, higher number means more details. + This is useful to see a trace from rule evaluation + without having to see trace from all other searchers at the same time. + </p> + + + + <h2 id="other">Other</h2> + + <h3 id="recall">recall</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>Any allowed collection of recall terms</td></tr> + <tr><td>Default</td><td>No recall</td></tr> + </table> + <p> + Sets a recall parameter to be combined with the query. + This is identical to <a href="#model.filter">filter</a>, + except that recall terms are not exposed to the ranking framework and thus not ranked. + As such, one can not use unprefixed terms; they must either by positive or negative. + </p> + + + <h3 id="user">user</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>A string</td></tr> + <tr><td>Default</td><td>None</td></tr> + </table> + <p> + The id of the user making the query. The contents of the argument are made available to the search chain, + but it triggers no features in Vespa apart from being propagated to the access log. + </p> + + + <h3 id="nocachewrite">nocachewrite</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>Boolean</td></tr> + <tr><td>Default</td><td>False</td></tr> + </table> + <p>Set to true to avoid the result being written to cache when fetched.</p> + + + <h3 id="hitcountestimate">hitcountestimate</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>Boolean</td></tr> + <tr><td>Default</td><td>False</td></tr> + </table> + <p>Make this an estimation query. + No hits will be returned, and total hit count will be set to an estimate of what executing + the query as a normal query would give. + </p> + + <h3 id="metrics.ignore">metrics.ignore</h3> + <table class="table table-striped"> + <tr><td>Alias</td><td></td></tr> + <tr><td>Values</td><td>Boolean</td></tr> + <tr><td>Default</td><td>False</td></tr> + </table> + <p>Ignore metric collection for this query request, useful for warm up queries</p> + + + </div> + + </div> +</div> + +</body> +</html> diff --git a/container-search/src/main/java/com/yahoo/fs4/mplex/Backend.java b/container-search/src/main/java/com/yahoo/fs4/mplex/Backend.java index 2a90e746378..202ee94383f 100644 --- a/container-search/src/main/java/com/yahoo/fs4/mplex/Backend.java +++ b/container-search/src/main/java/com/yahoo/fs4/mplex/Backend.java @@ -354,6 +354,31 @@ public class Backend implements ConnectionFactory { } /** + * Attempt to establish a connection without sending messages and then + * return it to the pool. The assumption is that if the probing is + * successful, the connection will be used soon after. There should be + * minimal overhead since the connection is cached. + */ + public boolean probeConnection() { + if (shutdownInitiated) { + return false; + } + + FS4Connection connection = null; + try { + connection = getConnection(); + } catch (IOException ignored) { + // connection is null + } finally { + if (connection != null) { + returnConnection(connection); + } + } + + return connection != null; + } + + /** * This method should be used to ensure graceful shutdown of the backend. */ public void shutdown() { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java index 08dcbe17db2..f68bb718c8d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java @@ -2,9 +2,9 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableMap; +import com.yahoo.fs4.mplex.Backend; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.dispatch.CloseableInvoker; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InterleavedFillInvoker; import com.yahoo.search.dispatch.InterleavedSearchInvoker; @@ -12,7 +12,6 @@ import com.yahoo.search.dispatch.SearchCluster; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.result.Hit; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -43,31 +42,64 @@ public class FS4InvokerFactory { } public SearchInvoker getSearchInvoker(Query query, SearchCluster.Node node) { - return new FS4SearchInvoker(searcher, query, fs4ResourcePool, node); + Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); + return new FS4SearchInvoker(searcher, query, backend.openChannel(), node); } + /** + * Create a {@link SearchInvoker} for a list of content nodes. + * + * @param query the search query being processed + * @param nodes pre-selected list of content nodes + * @return Optional containing the SearchInvoker or <i>empty</i> if some node in the list is invalid + */ public Optional<SearchInvoker> getSearchInvoker(Query query, List<SearchCluster.Node> nodes) { - return getInvoker(nodes, node -> getSearchInvoker(query, node), InterleavedSearchInvoker::new); + Map<Integer, SearchInvoker> invokers = new HashMap<>(); + for (SearchCluster.Node node : nodes) { + if (node.isWorking()) { + Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); + if (backend.probeConnection()) { + invokers.put(node.key(), new FS4SearchInvoker(searcher, query, backend.openChannel(), node)); + } else { + return Optional.empty(); + } + } + } + if (invokers.size() == 1) { + return Optional.of(invokers.values().iterator().next()); + } else { + return Optional.of(new InterleavedSearchInvoker(invokers)); + } } public FillInvoker getFillInvoker(Query query, SearchCluster.Node node) { return new FS4FillInvoker(searcher, query, fs4ResourcePool, node.hostname(), node.fs4port(), node.key()); } + /** + * Create a {@link FillInvoker} for a the hits in a {@link Result}. + * + * @param result the Result containing hits that need to be filled + * @return Optional containing the FillInvoker or <i>empty</i> if some hit is from an unknown content node + */ public Optional<FillInvoker> getFillInvoker(Result result) { Collection<Integer> requiredNodes = requiredFillNodes(result); - List<SearchCluster.Node> nodes = new ArrayList<>(requiredNodes.size()); + Query query = result.getQuery(); + Map<Integer, FillInvoker> invokers = new HashMap<>(); for (Integer distKey : requiredNodes) { SearchCluster.Node node = nodesByKey.get(distKey); if (node == null) { return Optional.empty(); } - nodes.add(node); + invokers.put(distKey, getFillInvoker(query, node)); } - Query query = result.getQuery(); - return getInvoker(nodes, node -> getFillInvoker(query, node), InterleavedFillInvoker::new); + if (invokers.size() == 1) { + return Optional.of(invokers.values().iterator().next()); + } else { + return Optional.of(new InterleavedFillInvoker(invokers)); + } } private static Collection<Integer> requiredFillNodes(Result result) { @@ -81,40 +113,4 @@ public class FS4InvokerFactory { } return requiredNodes; } - - @FunctionalInterface - private interface InvokerConstructor<INVOKER> { - INVOKER construct(SearchCluster.Node node); - } - - @FunctionalInterface - private interface ClusterInvokerConstructor<CLUSTERINVOKER extends INVOKER, INVOKER> { - CLUSTERINVOKER construct(Map<Integer, INVOKER> subinvokers); - } - - /* Get an invocation object for the provided collection of nodes. If only one - node is used, only the single-node invoker is used. For multiple nodes, each - gets a single-node invoker and they are all wrapped into a cluster invoker. - The functional interfaces are used to allow code reuse with SearchInvokers - and FillInvokers even though they don't share much class hierarchy. */ - private <INVOKER extends CloseableInvoker, CLUSTERINVOKER extends INVOKER> Optional<INVOKER> getInvoker( - Collection<SearchCluster.Node> nodes, InvokerConstructor<INVOKER> singleNodeCtor, - ClusterInvokerConstructor<CLUSTERINVOKER, INVOKER> clusterCtor) { - if (nodes.size() == 1) { - SearchCluster.Node node = nodes.iterator().next(); - return Optional.of(singleNodeCtor.construct(node)); - } else { - Map<Integer, INVOKER> nodeInvokers = new HashMap<>(); - for (SearchCluster.Node node : nodes) { - if (node.isWorking()) { - nodeInvokers.put(node.key(), singleNodeCtor.construct(node)); - } - } - if (nodeInvokers.size() == 1) { - return Optional.of(nodeInvokers.values().iterator().next()); - } else { - return Optional.of(clusterCtor.construct(nodeInvokers)); - } - } - } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java index ac48aef7063..dc8cd53e638 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java @@ -39,12 +39,11 @@ public class FS4SearchInvoker extends SearchInvoker { private Query query = null; private QueryPacket queryPacket = null; - public FS4SearchInvoker(VespaBackEndSearcher searcher, Query query, FS4ResourcePool fs4ResourcePool, SearchCluster.Node node) { + public FS4SearchInvoker(VespaBackEndSearcher searcher, Query query, FS4Channel channel, SearchCluster.Node node) { this.searcher = searcher; this.node = Optional.of(node); + this.channel = channel; - Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); - this.channel = backend.openChannel(); channel.setQuery(query); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 2fdb10067ff..235e7af09d2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -14,9 +14,11 @@ import com.yahoo.search.dispatch.SearchPath.InvalidSearchPathException; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; /** * A dispatcher communicates with search nodes to perform queries and fill hits. @@ -42,14 +44,15 @@ public class Dispatcher extends AbstractComponent { public Dispatcher(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { this.searchCluster = new SearchCluster(dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus); - this.loadBalancer = new LoadBalancer(searchCluster); + this.loadBalancer = new LoadBalancer(searchCluster, + dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.rpcResourcePool = new RpcResourcePool(dispatchConfig); } /** For testing */ public Dispatcher(Map<Integer, Client.NodeConnection> nodeConnections, Client client) { this.searchCluster = null; - this.loadBalancer = new LoadBalancer(searchCluster); + this.loadBalancer = new LoadBalancer(searchCluster, true); this.rpcResourcePool = new RpcResourcePool(client, nodeConnections); } @@ -120,19 +123,37 @@ public class Dispatcher extends AbstractComponent { return invokerFactory.supply(query, Arrays.asList(node)); } - Optional<SearchCluster.Group> groupInCluster = loadBalancer.takeGroupForQuery(query); - if (!groupInCluster.isPresent()) { - return Optional.empty(); - } - SearchCluster.Group group = groupInCluster.get(); - query.trace(false, 2, "Dispatching internally to ", group); - - Optional<SearchInvoker> invoker = invokerFactory.supply(query, group.nodes()); - if (invoker.isPresent()) { - invoker.get().teardown(() -> loadBalancer.releaseGroup(group)); - } else { - loadBalancer.releaseGroup(group); + Set<Integer> tried = null; + int max = searchCluster.groups().size(); + for (int attempt = 0; attempt < max; attempt++) { + Optional<SearchCluster.Group> groupInCluster = loadBalancer.takeGroupForQuery(query); + if (! groupInCluster.isPresent()) { + // No groups available + break; + } + SearchCluster.Group group = groupInCluster.get(); + if (tried != null && tried.contains(group.id())) { + // bail out: LB is offering a previously discarded group + loadBalancer.releaseGroup(group); + break; + } + + Optional<SearchInvoker> invoker = invokerFactory.supply(query, group.nodes()); + if (invoker.isPresent()) { + query.trace(false, 2, "Dispatching internally to ", group); + invoker.get().teardown(() -> loadBalancer.releaseGroup(group)); + return invoker; + } else { + // invoker could not be produced (likely connectivity issue) + searchCluster.groupConnectionFailure(group); + loadBalancer.releaseGroup(group); + if (tried == null) { + tried = new HashSet<>(); + } + tried.add(group.id()); + } } - return invoker; + + return Optional.empty(); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 9eac9b9b63d..64e38a488ab 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -19,24 +19,27 @@ import java.util.logging.Logger; */ public class LoadBalancer { // The implementation here is a simplistic least queries in flight + round-robin load balancer - // TODO: consider the options in com.yahoo.vespa.model.content.TuningDispatch private static final Logger log = Logger.getLogger(LoadBalancer.class.getName()); private final List<GroupSchedule> scoreboard; private int needle = 0; - public LoadBalancer(SearchCluster searchCluster) { + public LoadBalancer(SearchCluster searchCluster, boolean roundRobin) { if (searchCluster == null) { this.scoreboard = null; return; } this.scoreboard = new ArrayList<>(searchCluster.groups().size()); - for (Group group : searchCluster.groups().values()) { + for (Group group : searchCluster.orderedGroups()) { scoreboard.add(new GroupSchedule(group)); } - Collections.shuffle(scoreboard); + + if(! roundRobin) { + // TODO - More randomness could be desirable + Collections.shuffle(scoreboard); + } } /** @@ -74,16 +77,18 @@ public class LoadBalancer { private Optional<Group> allocateNextGroup() { synchronized (this) { GroupSchedule bestSchedule = null; + int bestIndex = needle; int index = needle; for (int i = 0; i < scoreboard.size(); i++) { GroupSchedule sched = scoreboard.get(index); if (sched.isPreferredOver(bestSchedule)) { bestSchedule = sched; + bestIndex = index; } index = nextScoreboardIndex(index); } - needle = nextScoreboardIndex(needle); + needle = nextScoreboardIndex(bestIndex); Group ret = null; if (bestSchedule != null) { @@ -118,9 +123,18 @@ public class LoadBalancer { if (other == null) { return true; } - if (! group.hasSufficientCoverage()) { - return false; + + // different coverage + if (this.group.hasSufficientCoverage() != other.group.hasSufficientCoverage()) { + if (! this.group.hasSufficientCoverage()) { + // this doesn't have coverage, other does + return false; + } else { + // other doesn't have coverage, this does + return true; + } } + return this.score < other.score; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java index 0d50702acfd..e26dd5648eb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java @@ -46,7 +46,9 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { private static final Logger log = Logger.getLogger(SearchCluster.class.getName()); /** The min active docs a group must have to be considered up, as a % of the average active docs of the other groups */ - private double minActivedocsCoveragePercentage; + private final double minActivedocsCoveragePercentage; + private final double minGroupCoverage; + private final int maxNodesDownPerGroup; private final int size; private final ImmutableMap<Integer, Group> groups; private final ImmutableMultimap<String, Node> nodesByHost; @@ -67,15 +69,16 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { // Only needed until query requests are moved to rpc private final FS4ResourcePool fs4ResourcePool; - public SearchCluster(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, - int containerClusterSize, VipStatus vipStatus) { - this(dispatchConfig.minActivedocsPercentage(), toNodes(dispatchConfig), fs4ResourcePool, - containerClusterSize, vipStatus); + public SearchCluster(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { + this(dispatchConfig.minActivedocsPercentage(), dispatchConfig.minGroupCoverage(), dispatchConfig.maxNodesDownPerGroup(), + toNodes(dispatchConfig), fs4ResourcePool, containerClusterSize, vipStatus); } - public SearchCluster(double minActivedocsCoverage, List<Node> nodes, FS4ResourcePool fs4ResourcePool, - int containerClusterSize, VipStatus vipStatus) { + public SearchCluster(double minActivedocsCoverage, double minGroupCoverage, int maxNodesDownPerGroup, List<Node> nodes, FS4ResourcePool fs4ResourcePool, + int containerClusterSize, VipStatus vipStatus) { this.minActivedocsCoveragePercentage = minActivedocsCoverage; + this.minGroupCoverage = minGroupCoverage; + this.maxNodesDownPerGroup = maxNodesDownPerGroup; this.size = nodes.size(); this.fs4ResourcePool = fs4ResourcePool; this.vipStatus = vipStatus; @@ -153,6 +156,9 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { /** Returns the groups of this cluster as an immutable map indexed by group id */ public ImmutableMap<Integer, Group> groups() { return groups; } + /** Returns the groups of this cluster as an immutable list in introduction order */ + public ImmutableList<Group> orderedGroups() { return orderedGroups; } + /** Returns the n'th (zero-indexed) group in the cluster if possible */ public Optional<Group> group(int n) { if (orderedGroups.size() > n) { @@ -210,6 +216,10 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { vipStatus.removeFromRotation(this); } + public void groupConnectionFailure(Group group) { + group.setHasSufficientCoverage(false); // will be reset after next ping iteration + } + private void updateSufficientCoverage(Group group, boolean sufficientCoverage) { // update VIP status if we direct dispatch to this group and coverage status changed if (usesDirectDispatchTo(group) && sufficientCoverage != group.hasSufficientCoverage()) { @@ -254,27 +264,54 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { */ @Override public void pingIterationCompleted() { + int numGroups = orderedGroups.size(); + if (numGroups == 1) { + Group group = groups.values().iterator().next(); + group.aggregateActiveDocuments(); + updateSufficientCoverage(group, true); // by definition + return; + } + // Update active documents per group and use it to decide if the group should be active - for (Group group : groups.values()) + + long[] activeDocumentsInGroup = new long[numGroups]; + long sumOfActiveDocuments = 0; + for(int i = 0; i < numGroups; i++) { + Group group = orderedGroups.get(i); group.aggregateActiveDocuments(); - if (groups.size() == 1) { - updateSufficientCoverage(groups.values().iterator().next(), true); // by definition - } else { - for (Group currentGroup : groups.values()) { - long sumOfAactiveDocumentsInOtherGroups = 0; - for (Group otherGroup : groups.values()) - if (otherGroup != currentGroup) - sumOfAactiveDocumentsInOtherGroups += otherGroup.getActiveDocuments(); - long averageDocumentsInOtherGroups = sumOfAactiveDocumentsInOtherGroups / (groups.size() - 1); - if (averageDocumentsInOtherGroups == 0) - updateSufficientCoverage(currentGroup, true); // no information about any group; assume coverage - else - updateSufficientCoverage(currentGroup, - 100 * (double) currentGroup.getActiveDocuments() / averageDocumentsInOtherGroups > minActivedocsCoveragePercentage); + activeDocumentsInGroup[i] = group.getActiveDocuments(); + sumOfActiveDocuments += activeDocumentsInGroup[i]; + } + + for (int i = 0; i < numGroups; i++) { + Group group = orderedGroups.get(i); + long activeDocuments = activeDocumentsInGroup[i]; + long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); + boolean sufficientCoverage = true; + + if (averageDocumentsInOtherGroups > 0) { + double coverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; + sufficientCoverage = coverage >= minActivedocsCoveragePercentage; } + if (sufficientCoverage) { + sufficientCoverage = isNodeCoverageSufficient(group); + } + updateSufficientCoverage(group, sufficientCoverage); } } + private boolean isNodeCoverageSufficient(Group group) { + int nodesUp = 0; + for (Node node : group.nodes()) { + if (node.isWorking()) { + nodesUp++; + } + } + int nodes = group.nodes().size(); + int nodesAllowedDown = maxNodesDownPerGroup + (int) (((double) nodes * (100.0 - minGroupCoverage)) / 100.0); + return nodesUp + nodesAllowedDown >= nodes; + } + private Pong getPong(FutureTask<Pong> futurePong, Node node) { try { return futurePong.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); @@ -346,8 +383,11 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { void aggregateActiveDocuments() { long activeDocumentsInGroup = 0; - for (Node node : nodes) - activeDocumentsInGroup += node.getActiveDocuments(); + for (Node node : nodes) { + if (node.isWorking()) { + activeDocumentsInGroup += node.getActiveDocuments(); + } + } activeDocuments.set(activeDocumentsInGroup); } diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/qr-start.def index 948f360ea63..d29ebc8a826 100644 --- a/container-search/src/main/resources/configdefinitions/qr-start.def +++ b/container-search/src/main/resources/configdefinitions/qr-start.def @@ -9,7 +9,7 @@ jvm.server bool default=true restart jvm.verbosegc bool default=true restart ## Garbage Collection tuning parameters -jvm.gcopts string default="" restart +jvm.gcopts string default="-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1" restart ## Heap size (in megabytes) for the Java VM jvm.heapsize int default=1536 restart diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index b08a3a73a01..9311ddab3c6 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -22,8 +22,8 @@ public class LoadBalancerTest { @Test public void requreThatLoadBalancerServesSingleNodeSetups() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(new Query()); Group group = grp.orElseGet(() -> { @@ -36,8 +36,8 @@ public class LoadBalancerTest { public void requreThatLoadBalancerServesMultiGroupSetups() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(new Query()); Group group = grp.orElseGet(() -> { @@ -52,8 +52,8 @@ public class LoadBalancerTest { Node n2 = new SearchCluster.Node(1, "test-node2", 1, 0); Node n3 = new SearchCluster.Node(0, "test-node3", 0, 1); Node n4 = new SearchCluster.Node(1, "test-node4", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2, n3, n4), null, 2, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2, n3, n4), null, 2, null); + LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(new Query()); assertThat(grp.isPresent(), is(true)); @@ -63,8 +63,8 @@ public class LoadBalancerTest { public void requreThatLoadBalancerReturnsDifferentGroups() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster, true); // get first group Optional<Group> grp = lb.takeGroupForQuery(new Query()); @@ -83,8 +83,8 @@ public class LoadBalancerTest { public void requreThatLoadBalancerReturnsGroupWithShortestQueue() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster, true); // get first group Optional<Group> grp = lb.takeGroupForQuery(new Query()); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 89b416f3293..ee903fd3fa0 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -19,7 +19,7 @@ public class MockSearchCluster extends SearchCluster { private final ImmutableMultimap<String, Node> nodesByHost; public MockSearchCluster(int groups, int nodesPerGroup) { - super(100, Collections.emptyList(), null, 1, null); + super(100, 100, 0, Collections.emptyList(), null, 1, null); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java index e95e97527da..aba3b5f3ab7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.athenz; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; /** @@ -10,12 +10,9 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; */ public interface AthenzClientFactory { - AthenzIdentity getControllerIdentity(); + AthenzService getControllerIdentity(); - ZmsClient createZmsClientWithServicePrincipal(); - - ZtsClient createZtsClientWithServicePrincipal(); - - ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken); + ZmsClient createZmsClient(); + ZtsClient createZtsClient(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java deleted file mode 100644 index 3630748b10a..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.athenz; - -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; - -import java.util.List; - -/** - * @author bjorncs - */ -public interface ZmsClient { - - void createTenant(AthenzDomain tenantDomain); - - void deleteTenant(AthenzDomain tenantDomain); - - void addApplication(AthenzDomain tenantDomain, ApplicationId applicationName); - - void deleteApplication(AthenzDomain tenantDomain, ApplicationId applicationName); - - boolean hasApplicationAccess(AthenzIdentity athenzIdentity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName); - - boolean hasTenantAdminAccess(AthenzIdentity athenzIdentity, AthenzDomain tenantDomain); - - boolean hasHostedOperatorAccess(AthenzIdentity identity); - - // Used before vespa tenancy is established for the domain. - boolean isDomainAdmin(AthenzIdentity athenzIdentity, AthenzDomain domain); - - List<AthenzDomain> getDomainList(String prefix); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsException.java deleted file mode 100644 index 31e9e549c08..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsException.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.athenz; - -/** - * @author bjorncs - */ -public class ZmsException extends RuntimeException { - - private final int code; - - public ZmsException(int code, Throwable cause) { - super(cause.getMessage(), cause); - this.code = code; - } - - public ZmsException(int code, String message) { - super(message); - this.code = code; - } - - public int getCode() { - return code; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 4255c24c977..1dff20c77be 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -27,16 +27,13 @@ public interface ConfigServer { PrepareResponse prepareResponse(); } - // TODO: Deprecated, remove when implementations have been removed - default PreparedApplication prepare(DeploymentId applicationInstance, DeployOptions deployOptions, Set<String> rotationCnames, Set<String> rotationNames, byte[] content) { - return deploy(applicationInstance, deployOptions, rotationCnames, rotationNames, content); - } + PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<String> rotationNames, byte[] content); - PreparedApplication deploy(DeploymentId applicationInstance, DeployOptions deployOptions, Set<String> rotationCnames, Set<String> rotationNames, byte[] content); + void restart(DeploymentId deployment, Optional<Hostname> hostname); - void restart(DeploymentId applicationInstance, Optional<Hostname> hostname) throws NoInstanceException; + void deactivate(DeploymentId deployment) throws NoInstanceException; - void deactivate(DeploymentId applicationInstance) throws NoInstanceException; + boolean isSuspended(DeploymentId deployment); ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index e7ef3e52eb5..f35c5b1c310 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -34,6 +34,9 @@ public interface ZoneRegistry { /** Returns the URI for the config server VIP in the given zone, or Optional.empty() if no VIP exists */ default Optional<URI> getConfigServerVipUri(ZoneId zoneId) { return Optional.empty(); } + /** Returns all possible API endpoints of all known config servers and config server VIPs in the given zone */ + List<URI> getConfigServerApiUris(ZoneId zoneId); + /** Returns a URL with the logs for the given deployment, if logging is configured for its zone */ Optional<URI> getLogServerUri(DeploymentId deploymentId); diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 44954e19fea..b15a561c1d2 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -130,50 +130,6 @@ <version>1.6</version> </dependency> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zms-java-client</artifactId> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <!-- Exclude all Jersey bundles provided by JDisc --> - <exclusion> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-client</artifactId> - </exclusion> - <exclusion> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-json-jackson</artifactId> - </exclusion> - <!-- BouncyCastle is not bundled due to class loading issues - when security provider is registered from inside a OSGi bundle --> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>bcpkix-jdk15on</artifactId> - </exclusion> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-jdk15on</artifactId> - </exclusion> - <!--Exclude all Jackson bundles provided by JDisc --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - </exclusions> - </dependency> - <!-- test --> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 366be35fe15..cd063e8e14b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -192,7 +192,7 @@ public class Application { /** Returns the global rotation dns name, if present */ public Optional<GlobalDnsName> globalDnsName(SystemName system) { - return rotation.map(rotation -> new GlobalDnsName(id, rotation, system)); + return rotation.map(ignored -> new GlobalDnsName(id, system)); } /** Returns the status of the global rotation assigned to this. Wil be empty if this does not have a global rotation. */ @@ -205,7 +205,6 @@ public class Application { // Rotation status only contains VIP host names, one per zone in the system. The only way to map VIP hostname to // this deployment, and thereby determine rotation status, is to check if VIP hostname contains the // deployment's environment and region. - // TODO: change this map to be indexed by zones, then? return rotationStatus.entrySet().stream() .filter(kv -> kv.getKey().value().contains(deployment.zone().value())) .map(Map.Entry::getValue) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 79bc72f950d..bc48051c111 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -9,7 +9,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; @@ -20,8 +20,8 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; @@ -43,6 +43,7 @@ import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; @@ -93,7 +94,7 @@ public class ApplicationController { private final ArtifactRepository artifactRepository; private final ApplicationStore applicationStore; private final RotationRepository rotationRepository; - private final AthenzClientFactory zmsClientFactory; + private final ZmsClientFacade zmsClient; private final NameService nameService; private final ConfigServer configServer; private final RoutingGenerator routingGenerator; @@ -108,7 +109,7 @@ public class ApplicationController { RoutingGenerator routingGenerator, BuildService buildService, Clock clock) { this.controller = controller; this.curator = curator; - this.zmsClientFactory = zmsClientFactory; + this.zmsClient = new ZmsClientFacade(zmsClientFactory.createZmsClient(), zmsClientFactory.getControllerIdentity()); this.nameService = nameService; this.configServer = configServer; this.routingGenerator = routingGenerator; @@ -250,7 +251,7 @@ public class ApplicationController { * * @throws IllegalArgumentException if the application already exists */ - public Application createApplication(ApplicationId id, Optional<NToken> token) { + public Application createApplication(ApplicationId id, Optional<OktaAccessToken> token) { if ( ! (id.instance().isDefault())) // TODO: Support instances properly throw new IllegalArgumentException("Only the instance name 'default' is supported at the moment"); if (id.instance().isTester()) @@ -269,11 +270,10 @@ public class ApplicationController { throw new IllegalArgumentException("Could not create '" + id + "': Application " + dashToUnderscore(id) + " already exists"); if (id.instance().isDefault() && tenant.get() instanceof AthenzTenant) { // Only create the athenz application for "default" instances. if ( ! token.isPresent()) - throw new IllegalArgumentException("Could not create '" + id + "': No NToken provided"); + throw new IllegalArgumentException("Could not create '" + id + "': No Okta Access Token provided"); - ZmsClient zmsClient = zmsClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()); zmsClient.addApplication(((AthenzTenant) tenant.get()).domain(), - new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); + new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value()), token.get()); } LockedApplication application = new LockedApplication(new Application(id, clock.instant()), lock); store(application); @@ -323,7 +323,7 @@ public class ApplicationController { ? triggered.sourceApplication().orElse(triggered.application()) : triggered.application(); - if (application.get().deploymentJobs().builtInternally()) { + if (application.get().deploymentJobs().deployedInternally()) { applicationPackage = new ApplicationPackage(applicationStore.getApplicationPackage(application.get().id(), applicationVersion.id())); } else { applicationPackage = new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())); @@ -332,7 +332,7 @@ public class ApplicationController { } // Update application with information from application package - if ( ! preferOldestVersion && ! application.get().deploymentJobs().builtInternally()) + if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally()) application = storeWithUpdatedConfig(application, applicationPackage); // Assign global rotation @@ -531,7 +531,7 @@ public class ApplicationController { * @throws IllegalArgumentException if the application has deployments or the caller is not authorized * @throws NotExistsException if no instances of the application exist */ - public void deleteApplication(ApplicationId applicationId, Optional<NToken> token) { + public void deleteApplication(ApplicationId applicationId, Optional<OktaAccessToken> token) { // Find all instances of the application List<ApplicationId> instances = asList(applicationId.tenant()).stream() .map(Application::id) @@ -548,13 +548,12 @@ public class ApplicationController { Tenant tenant = controller.tenants().tenant(id.tenant()).get(); if (tenant instanceof AthenzTenant && ! token.isPresent()) - throw new IllegalArgumentException("Could not delete '" + application + "': No NToken provided"); + throw new IllegalArgumentException("Could not delete '" + application + "': No Okta Access Token provided"); // Only delete in Athenz once if (id.instance().isDefault() && tenant instanceof AthenzTenant) { - zmsClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()) - .deleteApplication(((AthenzTenant) tenant).domain(), - new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); + zmsClient.deleteApplication(((AthenzTenant) tenant).domain(), + new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value()), token.get()); } curator.removeApplication(id); @@ -602,11 +601,21 @@ public class ApplicationController { * @param hostname If non-empty, restart will only be scheduled for this host */ public void restart(DeploymentId deploymentId, Optional<Hostname> hostname) { + configServer.restart(deploymentId, hostname); + } + + /** + * Asks the config server whether this deployment is currently <i>suspended</i>: + * Not in a state where it should receive traffic. + */ + public boolean isSuspended(DeploymentId deploymentId) { try { - configServer.restart(deploymentId, hostname); + return configServer.isSuspended(deploymentId); } - catch (NoInstanceException e) { - throw new IllegalArgumentException("Could not restart " + deploymentId + ": No such deployment"); + catch (ConfigServerException e) { + if (e.getErrorCode() == ConfigServerException.ErrorCode.NOT_FOUND) + return false; + throw e; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 9e3c5a2d683..47e11af056d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.OsVersion; @@ -74,7 +75,7 @@ public class Controller extends AbstractComponent { private final ConfigServer configServer; private final MetricsService metricsService; private final Chef chef; - private final AthenzClientFactory athenzClientFactory; + private final ZmsClientFacade zmsClient; /** * Creates a controller @@ -114,7 +115,7 @@ public class Controller extends AbstractComponent { this.metricsService = Objects.requireNonNull(metricsService, "MetricsService cannot be null"); this.chef = Objects.requireNonNull(chef, "Chef cannot be null"); this.clock = Objects.requireNonNull(clock, "Clock cannot be null"); - this.athenzClientFactory = Objects.requireNonNull(athenzClientFactory, "AthenzClientFactory cannot be null"); + this.zmsClient = new ZmsClientFacade(athenzClientFactory.createZmsClient(), athenzClientFactory.getControllerIdentity()); jobController = new JobController(this, runDataStore, Objects.requireNonNull(testerCloud)); applicationController = new ApplicationController(this, curator, athenzClientFactory, @@ -144,7 +145,7 @@ public class Controller extends AbstractComponent { public JobController jobController() { return jobController; } public List<AthenzDomain> getDomainList(String prefix) { - return athenzClientFactory.createZmsClientWithServicePrincipal().getDomainList(prefix); + return zmsClient.getDomainList(prefix); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 1ae3e6a6577..f0e13349fbf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -3,13 +3,14 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -41,12 +42,16 @@ public class TenantController { private final Controller controller; private final CuratorDb curator; - private final AthenzClientFactory athenzClientFactory; + private final ZmsClientFacade zmsClient; + private final ZtsClient ztsClient; + private final AthenzService controllerIdentity; public TenantController(Controller controller, CuratorDb curator, AthenzClientFactory athenzClientFactory) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); this.curator = Objects.requireNonNull(curator, "curator must be non-null"); - this.athenzClientFactory = Objects.requireNonNull(athenzClientFactory, "athenzClientFactory must be non-null"); + this.controllerIdentity = athenzClientFactory.getControllerIdentity(); + this.zmsClient = new ZmsClientFacade(athenzClientFactory.createZmsClient(), controllerIdentity); + this.ztsClient = athenzClientFactory.createZtsClient(); // Update serialization format of all tenants Once.after(Duration.ofMinutes(1), () -> { @@ -84,13 +89,11 @@ public class TenantController { /** Returns a list of all tenants accessible by the given user */ public List<Tenant> asList(UserId user) { AthenzUser athenzUser = AthenzUser.fromUserId(user.id()); - try (ZtsClient ztsClient = athenzClientFactory.createZtsClientWithServicePrincipal()) { - Set<AthenzDomain> userDomains = new HashSet<>(ztsClient.getTenantDomains(athenzClientFactory.getControllerIdentity(), athenzUser, "admin")); + Set<AthenzDomain> userDomains = new HashSet<>(ztsClient.getTenantDomains(controllerIdentity, athenzUser, "admin")); return asList().stream() .filter(tenant -> isUser(tenant, user) || userDomains.stream().anyMatch(domain -> inDomain(tenant, domain))) .collect(Collectors.toList()); - } } /** @@ -124,7 +127,7 @@ public class TenantController { } /** Create an Athenz tenant */ - public void create(AthenzTenant tenant, NToken token) { + public void create(AthenzTenant tenant, OktaAccessToken token) { try (Lock lock = lock(tenant.name())) { requireNonExistent(tenant.name()); AthenzDomain domain = tenant.domain(); @@ -136,7 +139,7 @@ public class TenantController { existingTenantWithDomain.get().name().value() + "'"); } - athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token).createTenant(domain); + zmsClient.createTenant(domain, token); curator.writeTenant(tenant); } } @@ -169,7 +172,7 @@ public class TenantController { } /** Update Athenz domain for tenant. Returns the updated tenant which must be explicitly stored */ - public LockedTenant withDomain(LockedTenant tenant, AthenzDomain newDomain, NToken token) { + public LockedTenant withDomain(LockedTenant tenant, AthenzDomain newDomain, OktaAccessToken token) { AthenzDomain existingDomain = tenant.get().domain(); if (existingDomain.equals(newDomain)) return tenant; Optional<Tenant> existingTenantWithNewDomain = tenantIn(newDomain); @@ -177,12 +180,11 @@ public class TenantController { throw new IllegalArgumentException("Could not set domain of " + tenant + " to '" + newDomain + "':" + existingTenantWithNewDomain.get() + " already has this domain"); - ZmsClient zmsClient = athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token); - zmsClient.createTenant(newDomain); + zmsClient.createTenant(newDomain, token); List<Application> applications = controller.applications().asList(tenant.get().name()); - applications.forEach(a -> zmsClient.addApplication(newDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value()))); - applications.forEach(a -> zmsClient.deleteApplication(existingDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value()))); - zmsClient.deleteTenant(existingDomain); + applications.forEach(a -> zmsClient.addApplication(newDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value()), token)); + applications.forEach(a -> zmsClient.deleteApplication(existingDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value()), token)); + zmsClient.deleteTenant(existingDomain, token); log.info("Set Athenz domain for '" + tenant + "' from '" + existingDomain + "' to '" + newDomain + "'"); return tenant.with(newDomain); @@ -196,10 +198,10 @@ public class TenantController { } /** Delete an Athenz tenant */ - public void deleteTenant(AthenzTenant tenant, NToken nToken) { + public void deleteTenant(AthenzTenant tenant, OktaAccessToken token) { try (Lock lock = lock(tenant.name())) { deleteTenant(tenant.name()); - athenzClientFactory.createZmsClientWithAuthorizedServiceToken(nToken).deleteTenant(tenant.domain()); + zmsClient.deleteTenant(tenant.domain(), token); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index e46df3c5a4a..c7e7165cb91 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -139,14 +139,6 @@ public class ApplicationList { .anyMatch(d -> d.version().isBefore(version)))); } - /** - * Returns the subset of applications which are not pull requests: - * Pull requests changes the application instance name to (default-pr)?[pull-request-number] - */ - public ApplicationList notPullRequest() { - return listOf(list.stream().filter(a -> ! a.id().instance().value().matches("^(default-pr)?\\d+$"))); - } - /** Returns the subset of applications which have a project ID */ public ApplicationList withProjectId() { return listOf(list.stream().filter(a -> a.deploymentJobs().projectId().isPresent())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index c099e856d04..1016a5adf65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -23,7 +23,7 @@ public class Deployment { private final ApplicationVersion applicationVersion; private final Version version; private final Instant deployTime; - private final Map<Id, ClusterUtilization> clusterUtils; + private final Map<Id, ClusterUtilization> clusterUtilization; private final Map<Id, ClusterInfo> clusterInfo; private final DeploymentMetrics metrics; private final DeploymentActivity activity; @@ -34,14 +34,14 @@ public class Deployment { } public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, - Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo, + Map<Id, ClusterUtilization> clusterUtilization, Map<Id, ClusterInfo> clusterInfo, DeploymentMetrics metrics, DeploymentActivity activity) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null"); - this.clusterUtils = Objects.requireNonNull(clusterUtils, "clusterUtils cannot be null"); + this.clusterUtilization = Objects.requireNonNull(clusterUtilization, "clusterUtilization cannot be null"); this.clusterInfo = Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null"); this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); this.activity = Objects.requireNonNull(activity, "activity cannot be null"); @@ -74,11 +74,11 @@ public class Deployment { /** Returns utilization of the clusters allocated to this */ public Map<Id, ClusterUtilization> clusterUtils() { - return clusterUtils; + return clusterUtilization; } public Deployment recordActivityAt(Instant instant) { - return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, clusterInfo, metrics, + return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, activity.recordAt(instant, metrics)); } @@ -88,12 +88,12 @@ public class Deployment { } public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) { - return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, newClusterInfo, metrics, + return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, newClusterInfo, metrics, activity); } public Deployment withMetrics(DeploymentMetrics metrics) { - return new Deployment(zone, applicationVersion, version, deployTime, clusterUtils, clusterInfo, metrics, + return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, activity); } @@ -105,12 +105,12 @@ public class Deployment { public DeploymentCost calculateCost() { Map<String, ClusterCost> costClusters = new HashMap<>(); - for (Id clusterId : clusterUtils.keySet()) { + for (Id clusterId : clusterUtilization.keySet()) { // Only include cluster cost if we have both cluster utilization and cluster info if (clusterInfo.containsKey(clusterId)) { costClusters.put(clusterId.value(), new ClusterCost(clusterInfo.get(clusterId), - clusterUtils.get(clusterId))); + clusterUtilization.get(clusterId))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 65ba7e68d31..23826a47931 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -114,7 +114,7 @@ public class DeploymentJobs { public Optional<IssueId> issueId() { return issueId; } - public boolean builtInternally() { return builtInternally; } + public boolean deployedInternally() { return builtInternally; } private static OptionalLong requireId(OptionalLong id, String message) { Objects.requireNonNull(id, message); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java index 18ec15d35ac..91406bdb941 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.net.URI; @@ -24,7 +23,7 @@ public class GlobalDnsName { private final URI secureUrl; private final URI oathUrl; - public GlobalDnsName(ApplicationId application, RotationId id, SystemName system) { + public GlobalDnsName(ApplicationId application, SystemName system) { this.url = URI.create(String.format("http://%s%s.%s.%s:%d/", getSystemPart(system, "."), sanitize(application.application().value()), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ApplicationAction.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java index 3323cda89b3..8614414dc95 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ApplicationAction.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.athenz; +package com.yahoo.vespa.hosted.controller.athenz; /** * @author bjorncs diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/HostedAthenzIdentities.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java index bd385034a90..08dbdff13db 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/HostedAthenzIdentities.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/HostedAthenzIdentities.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.athenz; +package com.yahoo.vespa.hosted.controller.athenz; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzService; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java deleted file mode 100644 index 26cd9f2e9b8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; - -import com.google.inject.Inject; -import com.yahoo.jdisc.Response; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.security.athenz.AthenzPrincipalFilter; -import com.yahoo.jdisc.http.filter.security.athenz.AthenzPrincipalFilterConfig; -import com.yahoo.jdisc.http.filter.security.cors.CorsFilterConfig; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.api.NToken; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; -import com.yahoo.yolean.chain.After; - -import java.security.Principal; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.stream.Stream; - - -/** - * A variant of the {@link AthenzPrincipalFilter} to be used in combination with a cookie-based - * security filter for user authentication - * Assumes that the user authentication filter configured in the same filter chain and is configured to run before this filter. - * - * @author bjorncs - */ -// TODO Remove this filter once migrated to Okta -@After({"CorsPreflightRequestFilter", "BouncerFilter"}) -public class UserAuthWithAthenzPrincipalFilter extends AthenzPrincipalFilter { - - private static final Logger log = Logger.getLogger(UserAuthWithAthenzPrincipalFilter.class.getName()); - - private final String userAuthenticationPassThruAttribute; - private final String principalHeaderName; - - @Inject - public UserAuthWithAthenzPrincipalFilter(AthenzPrincipalFilterConfig filterConfig, AthenzConfig athenzConfig, CorsFilterConfig corsConfig) { - super(filterConfig, corsConfig); - this.userAuthenticationPassThruAttribute = athenzConfig.userAuthenticationPassThruAttribute(); - this.principalHeaderName = filterConfig.principalHeaderName(); - } - - @Override - public Optional<ErrorResponse> filterRequest(DiscFilterRequest request) { - if (request.getMethod().equals("OPTIONS")) return Optional.empty(); // Skip authentication on OPTIONS - required for Javascript CORS - - try { - switch (getUserAuthenticationResult(request)) { - case USER_COOKIE_MISSING: - case USER_COOKIE_ALTERNATIVE_MISSING: - return super.filterRequest(request); // Cookie-based authentication failed, delegate to Athenz - case USER_COOKIE_OK: - rewriteUserPrincipalToAthenz(request); - return Optional.empty(); // Authenticated using user cookie - case USER_COOKIE_INVALID: - return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, "Your user cookie is invalid (either expired, tampered or invalid ip)")); - default: - return Optional.empty(); - } - } catch (Exception e) { - log.log(LogLevel.WARNING, "Authentication failed: " + e.getMessage(), e); - return Optional.of(new ErrorResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage())); - } - } - - private UserAuthenticationResult getUserAuthenticationResult(DiscFilterRequest request) { - if (!request.containsAttribute(userAuthenticationPassThruAttribute)) { - throw new IllegalStateException("User authentication filter passthru attribute missing"); - } - Integer statusCode = (Integer) request.getAttribute(userAuthenticationPassThruAttribute); - return Stream.of(UserAuthenticationResult.values()) - .filter(uar -> uar.statusCode == statusCode) - .findAny() - .orElseThrow(() -> new IllegalStateException("Invalid status code: " + statusCode)); - } - - private void rewriteUserPrincipalToAthenz(DiscFilterRequest request) { - Principal userPrincipal = request.getUserPrincipal(); - log.log(LogLevel.DEBUG, () -> "Original user principal: " + userPrincipal.toString()); - UserId userId = new UserId(userPrincipal.getName()); - AthenzUser athenzIdentity = AthenzUser.fromUserId(userId.id()); - request.setRemoteUser(athenzIdentity.getFullName()); - NToken nToken = Optional.ofNullable(request.getHeader(principalHeaderName)).map(NToken::new).orElse(null); - request.setUserPrincipal(new AthenzPrincipal(athenzIdentity, nToken)); - } - - private enum UserAuthenticationResult { - USER_COOKIE_MISSING(0), - USER_COOKIE_OK(1), - USER_COOKIE_INVALID(-1), - USER_COOKIE_ALTERNATIVE_MISSING(-2); - - final int statusCode; - - UserAuthenticationResult(int statusCode) { - this.statusCode = statusCode; - } - - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 633c0470080..846c90a96f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -2,47 +2,33 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.inject.Inject; -import com.yahoo.athenz.auth.Principal; -import com.yahoo.athenz.auth.impl.PrincipalAuthority; -import com.yahoo.athenz.auth.impl.SimplePrincipal; -import com.yahoo.athenz.auth.token.PrincipalToken; -import com.yahoo.athenz.auth.util.Crypto; -import com.yahoo.athenz.zms.ZMSClient; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.client.zms.DefaultZmsClient; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import java.net.URI; -import java.security.PrivateKey; /** * @author bjorncs */ public class AthenzClientFactoryImpl implements AthenzClientFactory { - private final SecretStore secretStore; private final AthenzConfig config; - private final AthenzPrincipalAuthority athenzPrincipalAuthority; private final ServiceIdentityProvider identityProvider; @Inject - public AthenzClientFactoryImpl(SecretStore secretStore, ServiceIdentityProvider identityProvider, AthenzConfig config) { - this.secretStore = secretStore; + public AthenzClientFactoryImpl(ServiceIdentityProvider identityProvider, AthenzConfig config) { this.identityProvider = identityProvider; this.config = config; - this.athenzPrincipalAuthority = new AthenzPrincipalAuthority(config.principalHeaderName()); } @Override - public AthenzIdentity getControllerIdentity() { + public AthenzService getControllerIdentity() { return identityProvider.identity(); } @@ -50,52 +36,16 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { * @return A ZMS client instance with the service identity as principal. */ @Override - public ZmsClient createZmsClientWithServicePrincipal() { - return new ZmsClientImpl(new ZMSClient(config.zmsUrl(), identityProvider.getIdentitySslContext()), config); + public ZmsClient createZmsClient() { + return new DefaultZmsClient(URI.create(config.zmsUrl()), identityProvider); } /** * @return A ZTS client instance with the service identity as principal. */ @Override - public ZtsClient createZtsClientWithServicePrincipal() { + public ZtsClient createZtsClient() { return new DefaultZtsClient(URI.create(config.ztsUrl()), identityProvider); } - /** - * @return A ZMS client created with a dual principal representing both the tenant admin and the service identity. - */ - @Override - public ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken) { - PrincipalToken signedToken = new PrincipalToken(authorizedServiceToken.getRawToken()); - AthenzConfig.Service service = config.service(); - signedToken.signForAuthorizedService( - config.domain() + "." + service.name(), service.publicKeyId(), getServicePrivateKey()); - - Principal dualPrincipal = SimplePrincipal.create( - AthenzIdentities.USER_PRINCIPAL_DOMAIN.getName(), signedToken.getName(), signedToken.getSignedToken(), athenzPrincipalAuthority); - return new ZmsClientImpl(new ZMSClient(config.legacyZmsUrl(), dualPrincipal), config); - - } - - private PrivateKey getServicePrivateKey() { - AthenzConfig.Service service = config.service(); - String privateKey = secretStore.getSecret(service.privateKeySecretName(), service.privateKeyVersion()).trim(); - return Crypto.loadPrivateKey(privateKey); - } - - private static class AthenzPrincipalAuthority extends PrincipalAuthority { - private final String principalHeaderName; - - public AthenzPrincipalAuthority(String principalHeaderName) { - this.principalHeaderName = principalHeaderName; - } - - @Override - public String getHeader() { - return principalHeaderName; - } - } - - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientFacade.java new file mode 100644 index 00000000000..09619a33cc4 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientFacade.java @@ -0,0 +1,122 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.impl; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; +import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.OktaAccessToken; +import com.yahoo.vespa.athenz.client.zms.RoleAction; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * @author bjorncs + */ +public class ZmsClientFacade { + + private static final Logger log = Logger.getLogger(ZmsClientFacade.class.getName()); + private final ZmsClient zmsClient; + private final AthenzService service; + + public ZmsClientFacade(ZmsClient zmsClient, AthenzService identity) { + this.zmsClient = zmsClient; + this.service = identity; + } + + public void createTenant(AthenzDomain tenantDomain, OktaAccessToken token) { + log("createTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); + zmsClient.createTenancy(tenantDomain, service, token); + } + + public void deleteTenant(AthenzDomain tenantDomain, OktaAccessToken token) { + log("deleteTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); + zmsClient.deleteTenancy(tenantDomain, service, token); + } + + public void addApplication(AthenzDomain tenantDomain, ApplicationId applicationName, OktaAccessToken token) { + Set<RoleAction> tenantRoleActions = createTenantRoleActions(); + log("createProviderResourceGroup(" + + "tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s, roleActions=%s)", + tenantDomain, service.getDomain().getName(), service.getName(), applicationName, tenantRoleActions); + zmsClient.createProviderResourceGroup(tenantDomain, service, applicationName.id(), tenantRoleActions, token); + } + + public void deleteApplication(AthenzDomain tenantDomain, ApplicationId applicationName, OktaAccessToken token) { + log("deleteProviderResourceGroup(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", + tenantDomain, service.getDomain().getName(), service.getName(), applicationName); + zmsClient.deleteProviderResourceGroup(tenantDomain, service, applicationName.id(), token); + } + + public boolean hasApplicationAccess( + AthenzIdentity identity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName) { + return hasAccess( + action.name(), applicationResourceString(tenantDomain, applicationName), identity); + } + + public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { + return hasAccess(TenantAction._modify_.name(), tenantResourceString(tenantDomain), identity); + } + + public boolean hasHostedOperatorAccess(AthenzIdentity identity) { + return hasAccess("modify", service.getDomain().getName() + ":hosted-vespa", identity); + } + + /** + * Used when creating tenancies. As there are no tenancy policies at this point, + * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)} + */ + public boolean isDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { + log("getMembership(domain=%s, role=%s, principal=%s)", domain, "admin", identity); + return zmsClient.getMembership(new AthenzRole(domain, "admin"), identity); + } + + public List<AthenzDomain> getDomainList(String prefix) { + log.log(LogLevel.DEBUG, String.format("getDomainList(prefix=%s)", prefix)); + return zmsClient.getDomainList(prefix); + } + + private static Set<RoleAction> createTenantRoleActions() { + return Arrays.stream(ApplicationAction.values()) + .map(action -> new RoleAction(action.roleName, action.name())) + .collect(Collectors.toSet()); + } + + private boolean hasAccess(String action, String resource, AthenzIdentity identity) { + log("getAccess(action=%s, resource=%s, principal=%s)", action, resource, identity); + return zmsClient.hasAccess(AthenzResourceName.fromString(resource), action, identity); + } + + private static void log(String format, Object... args) { + log.log(LogLevel.DEBUG, String.format(format, args)); + } + + private String resourceStringPrefix(AthenzDomain tenantDomain) { + return String.format("%s:service.%s.tenant.%s", + service.getDomain().getName(), service.getName(), tenantDomain.getName()); + } + + private String tenantResourceString(AthenzDomain tenantDomain) { + return resourceStringPrefix(tenantDomain) + ".wildcard"; + } + + private String applicationResourceString(AthenzDomain tenantDomain, ApplicationId applicationName) { + return resourceStringPrefix(tenantDomain) + "." + "res_group" + "." + applicationName.id() + ".wildcard"; + } + + private enum TenantAction { + // This is meant to match only the '*' action of the 'admin' role. + // If needed, we can replace it with 'create', 'delete' etc. later. + _modify_ + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java deleted file mode 100644 index 6179d9891fd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.impl; - -import com.yahoo.athenz.zms.DomainList; -import com.yahoo.athenz.zms.ProviderResourceGroupRoles; -import com.yahoo.athenz.zms.Tenancy; -import com.yahoo.athenz.zms.TenantRoleAction; -import com.yahoo.athenz.zms.ZMSClient; -import com.yahoo.athenz.zms.ZMSClientException; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; -import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.Supplier; -import java.util.logging.Logger; - -import static java.util.stream.Collectors.toList; - -/** - * @author bjorncs - */ -public class ZmsClientImpl implements ZmsClient { - - private static final Logger log = Logger.getLogger(ZmsClientImpl.class.getName()); - private final ZMSClient zmsClient; - private final AthenzService service; - - ZmsClientImpl(ZMSClient zmsClient, AthenzConfig config) { - this.zmsClient = zmsClient; - this.service = new AthenzService(config.domain(), config.service().name()); - } - - @Override - public void createTenant(AthenzDomain tenantDomain) { - log("putTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); - runOrThrow(() -> { - Tenancy tenancy = new Tenancy() - .setDomain(tenantDomain.getName()) - .setService(service.getFullName()) - .setResourceGroups(Collections.emptyList()); - zmsClient.putTenancy(tenantDomain.getName(), service.getFullName(), /*auditref*/null, tenancy); - }); - } - - @Override - public void deleteTenant(AthenzDomain tenantDomain) { - log("deleteTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); - runOrThrow(() -> zmsClient.deleteTenancy(tenantDomain.getName(), service.getFullName(), /*auditref*/null)); - } - - @Override - public void addApplication(AthenzDomain tenantDomain, ApplicationId applicationName) { - List<TenantRoleAction> tenantRoleActions = createTenantRoleActions(); - log("putProviderResourceGroupRoles(" + - "tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s, roleActions=%s)", - tenantDomain, service.getDomain().getName(), service.getName(), applicationName, tenantRoleActions); - runOrThrow(() -> { - ProviderResourceGroupRoles resourceGroupRoles = new ProviderResourceGroupRoles() - .setDomain(service.getDomain().getName()) - .setService(service.getName()) - .setTenant(tenantDomain.getName()) - .setResourceGroup(applicationName.id()) - .setRoles(tenantRoleActions); - zmsClient.putProviderResourceGroupRoles( - tenantDomain.getName(), service.getDomain().getName(), service.getName(), - applicationName.id(), /*auditref*/null, resourceGroupRoles); - }); - } - - @Override - public void deleteApplication(AthenzDomain tenantDomain, ApplicationId applicationName) { - log("deleteProviderResourceGroupRoles(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", - tenantDomain, service.getDomain().getName(), service.getName(), applicationName); - runOrThrow(() -> { - zmsClient.deleteProviderResourceGroupRoles( - tenantDomain.getName(), service.getDomain().getName(), service.getName(), applicationName.id(), /*auditref*/null); - }); - } - - @Override - public boolean hasApplicationAccess( - AthenzIdentity identity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName) { - return hasAccess( - action.name(), applicationResourceString(tenantDomain, applicationName), identity); - } - - @Override - public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { - return hasAccess(TenantAction._modify_.name(), tenantResourceString(tenantDomain), identity); - } - - @Override - public boolean hasHostedOperatorAccess(AthenzIdentity identity) { - return getOrThrow(() -> hasAccess("modify", service.getDomain().getName() + ":hosted-vespa", identity)); - } - - /** - * Used when creating tenancies. As there are no tenancy policies at this point, - * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)} - */ - @Override - public boolean isDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { - log("getMembership(domain=%s, role=%s, principal=%s)", domain, "admin", identity); - return getOrThrow( - () -> zmsClient.getMembership(domain.getName(), "admin", identity.getFullName()).getIsMember()); - } - - @Override - public List<AthenzDomain> getDomainList(String prefix) { - log.log(LogLevel.DEBUG, String.format("getDomainList(prefix=%s)", prefix)); - return getOrThrow( - () -> { - DomainList domainList = zmsClient.getDomainList( - /*limit*/null, /*skip*/null, prefix, /*depth*/null, /*domain*/null, - /*productId*/ null, /*modifiedSince*/null); - return toAthenzDomains(domainList.getNames()); - }); - } - - private static List<TenantRoleAction> createTenantRoleActions() { - return Arrays.stream(ApplicationAction.values()) - .map(action -> new TenantRoleAction().setAction(action.name()).setRole(action.roleName)) - .collect(toList()); - } - - private static List<AthenzDomain> toAthenzDomains(List<String> domains) { - return domains.stream().map(AthenzDomain::new).collect(toList()); - } - - private boolean hasAccess(String action, String resource, AthenzIdentity identity) { - log("getAccess(action=%s, resource=%s, principal=%s)", action, resource, identity); - return getOrThrow( - () -> zmsClient.getAccess(action, resource, /*trustDomain*/null, identity.getFullName()) - .getGranted()); - } - - private static void log(String format, Object... args) { - log.log(LogLevel.DEBUG, String.format(format, args)); - } - - private static void runOrThrow(Runnable wrappedCode) { - try { - wrappedCode.run(); - } catch (ZMSClientException e) { - logWarning(e); - throw new ZmsException(e.getCode(), e); - } - } - - private static <T> T getOrThrow(Supplier<T> wrappedCode) { - try { - return wrappedCode.get(); - } catch (ZMSClientException e) { - logWarning(e); - throw new ZmsException(e.getCode(), e); - } - } - - private static void logWarning(ZMSClientException e) { - log.warning("Error from Athenz: " + e.getMessage()); - } - - private String resourceStringPrefix(AthenzDomain tenantDomain) { - return String.format("%s:service.%s.tenant.%s", - service.getDomain().getName(), service.getName(), tenantDomain.getName()); - } - - private String tenantResourceString(AthenzDomain tenantDomain) { - return resourceStringPrefix(tenantDomain) + ".wildcard"; - } - - private String applicationResourceString(AthenzDomain tenantDomain, ApplicationId applicationName) { - return resourceStringPrefix(tenantDomain) + "." + "res_group" + "." + applicationName.id() + ".wildcard"; - } - - private enum TenantAction { - // This is meant to match only the '*' action of the 'admin' role. - // If needed, we can replace it with 'create', 'delete' etc. later. - _modify_ - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java index 6f829113016..f9f449121e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java @@ -3,11 +3,9 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import java.util.logging.Level; @@ -36,28 +34,20 @@ public class AthenzClientFactoryMock extends AbstractComponent implements Athenz } @Override - public AthenzIdentity getControllerIdentity() { + public AthenzService getControllerIdentity() { return new AthenzService("vespa.hosting"); } @Override - public ZmsClient createZmsClientWithServicePrincipal() { - log("createZmsClientWithServicePrincipal()"); - return new ZmsClientMock(athenz); + public ZmsClient createZmsClient() { + return new ZmsClientMock(athenz, getControllerIdentity()); } @Override - public ZtsClient createZtsClientWithServicePrincipal() { - log("createZtsClientWithServicePrincipal()"); + public ZtsClient createZtsClient() { return new ZtsClientMock(athenz); } - @Override - public ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken) { - log("createZmsClientWithAuthorizedServiceToken(authorizedServiceToken='%s')", authorizedServiceToken); - return new ZmsClientMock(athenz); - } - private static void log(String format, Object... args) { log.log(Level.INFO, String.format(format, args)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java index 0a360184da9..a11426b9a23 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import java.util.ArrayList; import java.util.HashMap; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index 5e8674ce637..f7a8e702b06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -1,18 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.mock; -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; +import com.yahoo.vespa.athenz.api.AthenzResourceName; +import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.OktaAccessToken; +import com.yahoo.vespa.athenz.client.zms.RoleAction; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; +import com.yahoo.vespa.athenz.client.zms.ZmsClientException; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author bjorncs @@ -22,20 +30,24 @@ public class ZmsClientMock implements ZmsClient { private static final Logger log = Logger.getLogger(ZmsClientMock.class.getName()); private final AthenzDbMock athenz; + private final AthenzService controllerIdentity; + private static final Pattern TENANT_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.(?<tenantDomain>[\\w\\-_]+)\\..*"); + private static final Pattern APPLICATION_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.[\\w\\-_]+\\.res_group\\.(?<resourceGroup>[\\w\\-_]+)\\.wildcard"); - public ZmsClientMock(AthenzDbMock athenz) { + public ZmsClientMock(AthenzDbMock athenz, AthenzService controllerIdentity) { this.athenz = athenz; + this.controllerIdentity = controllerIdentity; } @Override - public void createTenant(AthenzDomain tenantDomain) { - log("createTenant(tenantDomain='%s')", tenantDomain); + public void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + log("createTenancy(tenantDomain='%s')", tenantDomain); getDomainOrThrow(tenantDomain, false).isVespaTenant = true; } @Override - public void deleteTenant(AthenzDomain tenantDomain) { - log("deleteTenant(tenantDomain='%s')", tenantDomain); + public void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + log("deleteTenancy(tenantDomain='%s')", tenantDomain); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, false); domain.isVespaTenant = false; domain.applications.clear(); @@ -43,55 +55,80 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void addApplication(AthenzDomain tenantDomain, ApplicationId applicationName) { - log("addApplication(tenantDomain='%s', applicationName='%s')", tenantDomain, applicationName); + public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { + log("createProviderResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); - if (!domain.applications.containsKey(applicationName)) { - domain.applications.put(applicationName, new AthenzDbMock.Application()); + ApplicationId applicationId = new ApplicationId(resourceGroup); + if (!domain.applications.containsKey(applicationId)) { + domain.applications.put(applicationId, new AthenzDbMock.Application()); } } @Override - public void deleteApplication(AthenzDomain tenantDomain, ApplicationId applicationName) { - log("addApplication(tenantDomain='%s', applicationName='%s')", tenantDomain, applicationName); - getDomainOrThrow(tenantDomain, true).applications.remove(applicationName); + public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token) { + log("deleteProviderResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); + getDomainOrThrow(tenantDomain, true).applications.remove(new ApplicationId(resourceGroup)); } @Override - public boolean hasApplicationAccess(AthenzIdentity identity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName) { - log("hasApplicationAccess(principal='%s', action='%s', tenantDomain='%s', applicationName='%s')", - identity, action, tenantDomain, applicationName); - AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); - AthenzDbMock.Application application = domain.applications.get(applicationName); - if (application == null) { - throw zmsException(400, "Application '%s' not found", applicationName); + public boolean getMembership(AthenzRole role, AthenzIdentity identity) { + if (role.roleName().equals("admin")) { + return getDomainOrThrow(role.domain(), false).admins.contains(identity); } - return isHostedOperator(identity) || domain.admins.contains(identity) || application.acl.get(action).contains(identity); + return false; } @Override - public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { - log("hasTenantAdminAccess(principal='%s', tenantDomain='%s')", identity, tenantDomain); - return isHostedOperator(identity) || isDomainAdmin(identity, tenantDomain) || - getDomainOrThrow(tenantDomain, true).tenantAdmins.contains(identity); + public List<AthenzDomain> getDomainList(String prefix) { + log("getDomainList()"); + return new ArrayList<>(athenz.domains.keySet()); } @Override - public boolean hasHostedOperatorAccess(AthenzIdentity identity) { - log("hasHostedOperatorAccess(identity='%s')", identity); - return isHostedOperator(identity); + public boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity) { + log("hasAccess(resource=%s, action=%s, identity=%s)", resource, action, identity); + if (resource.getDomain().equals(this.controllerIdentity.getDomain())) { + if (isHostedOperator(identity)) { + return true; + } + if (resource.getEntityName().startsWith("service.hosting.tenant.")) { + AthenzDomain tenantDomainName = getTenantDomain(resource); + AthenzDbMock.Domain tenantDomain = getDomainOrThrow(tenantDomainName, true); + if (tenantDomain.admins.contains(identity)) { + return true; + } + if (resource.getEntityName().contains(".res_group.")) { + ApplicationId applicationName = new ApplicationId(getResourceGroupName(resource)); + AthenzDbMock.Application application = tenantDomain.applications.get(applicationName); + if (application == null) { + throw zmsException(400, "Application '%s' not found", applicationName); + } + return application.acl.get(ApplicationAction.valueOf(action)).contains(identity); + } + return false; + } + return false; + } + return false; } @Override - public boolean isDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { - log("isDomainAdmin(principal='%s', domain='%s')", identity, domain); - return getDomainOrThrow(domain, false).admins.contains(identity); + public void close() {} + + private static AthenzDomain getTenantDomain(AthenzResourceName resource) { + Matcher matcher = TENANT_RESOURCE_PATTERN.matcher(resource.getEntityName()); + if (!matcher.matches()) { + throw new IllegalArgumentException(resource.toResourceNameString()); + } + return new AthenzDomain(matcher.group("tenantDomain")); } - @Override - public List<AthenzDomain> getDomainList(String prefix) { - log("getDomainList()"); - return new ArrayList<>(athenz.domains.keySet()); + private static String getResourceGroupName(AthenzResourceName resource) { + Matcher matcher = APPLICATION_RESOURCE_PATTERN.matcher(resource.getEntityName()); + if (!matcher.matches()) { + throw new IllegalArgumentException(resource.toResourceNameString()); + } + return matcher.group("resourceGroup"); } private AthenzDbMock.Domain getDomainOrThrow(AthenzDomain domainName, boolean verifyVespaTenant) { @@ -107,8 +144,8 @@ public class ZmsClientMock implements ZmsClient { return athenz.hostedOperators.contains(identity); } - private static ZmsException zmsException(int code, String message, Object... args) { - return new ZmsException(code, String.format(message, args)); + private static ZmsClientException zmsException(int code, String message, Object... args) { + return new ZmsClientException(code, String.format(message, args)); } private static void log(String format, Object... args) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 759c666e042..8bc61d6bc3b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -8,8 +8,10 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.JobState; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; @@ -106,9 +108,14 @@ public class DeploymentTrigger { triggering = JobRun.triggering(application.get().oldestDeployedPlatform().orElse(controller.systemVersion()), applicationVersion, Optional.empty(), Optional.empty(), "Application commit", clock.instant()); if (report.success()) { - if (acceptNewApplicationVersion(application.get())) + if (acceptNewApplicationVersion(application.get())) { application = application.withChange(application.get().change().with(applicationVersion)) .withOutstandingChange(Change.empty()); + if (application.get().deploymentJobs().deployedInternally()) + for (Run run : jobs.active()) + if (run.id().application().equals(report.applicationId())) + jobs.abort(run.id()); + } else application = application.withOutstandingChange(Change.of(applicationVersion)); } @@ -174,7 +181,7 @@ public class DeploymentTrigger { log.log(LogLevel.INFO, String.format("Triggering %s: %s", job, job.triggering)); try { applications().lockOrThrow(job.applicationId(), application -> { - if (application.get().deploymentJobs().builtInternally()) + if (application.get().deploymentJobs().deployedInternally()) jobs.start(job.applicationId(), job.jobType, new Versions(job.triggering.platform(), job.triggering.application(), job.triggering.sourcePlatform(), @@ -199,7 +206,7 @@ public class DeploymentTrigger { public List<JobType> forceTrigger(ApplicationId applicationId, JobType jobType, String user) { Application application = applications().require(applicationId); if (jobType == component) { - if (application.deploymentJobs().builtInternally()) + if (application.deploymentJobs().deployedInternally()) throw new IllegalArgumentException(applicationId + " has no component job we can trigger."); buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); @@ -268,7 +275,6 @@ public class DeploymentTrigger { /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ private List<Job> computeReadyJobs() { return ApplicationList.from(applications().asList()) - .notPullRequest() .withProjectId() .deploying() .idList().stream() @@ -296,7 +302,7 @@ public class DeploymentTrigger { for (Step step : steps.production()) { List<JobType> stepJobs = steps.toJobs(step); List<JobType> remainingJobs = stepJobs.stream().filter(job -> !isComplete(change, application, job)).collect(toList()); - if (!remainingJobs.isEmpty()) { // Step is incomplete; trigger remaining jobs if ready, or their test jobs if untested. + if (!remainingJobs.isEmpty()) { // Change is incomplete; trigger remaining jobs if ready, or their test jobs if untested. for (JobType job : remainingJobs) { Versions versions = Versions.from(change, application, deploymentFor(application, job), controller.systemVersion()); @@ -339,7 +345,12 @@ public class DeploymentTrigger { /** Returns whether given job should be triggered */ private boolean canTrigger(JobType job, Versions versions, Application application, List<JobType> parallelJobs) { if (jobStateOf(application, job) != idle) return false; - if (parallelJobs != null && !parallelJobs.containsAll(runningProductionJobs(application))) return false; + + // Are we already running jobs which are not in the set which can run in parallel with this? + if (parallelJobs != null && ! parallelJobs.containsAll(runningProductionJobs(application))) return false; + + // Are there another suspended deployment such that we shouldn't simultaneously change this? + if (job.isProduction() && isSuspendedInAnotherZone(application, job.zone(controller.system()))) return false; return triggerAt(clock.instant(), job, versions, application); } @@ -349,6 +360,15 @@ public class DeploymentTrigger { return canTrigger(job, versions, application, null); } + private boolean isSuspendedInAnotherZone(Application application, ZoneId zone) { + for (Deployment deployment : application.productionDeployments().values()) { + if ( ! deployment.zone().equals(zone) + && controller.applications().isSuspended(new DeploymentId(application.id(), deployment.zone()))) + return true; + } + return false; + } + /** Returns whether job can trigger at given instant */ public boolean triggerAt(Instant instant, JobType job, Versions versions, Application application) { Optional<JobStatus> jobStatus = application.deploymentJobs().statusOf(job); @@ -395,7 +415,7 @@ public class DeploymentTrigger { } private JobState jobStateOf(Application application, JobType jobType) { - if (application.deploymentJobs().builtInternally()) { + if (application.deploymentJobs().deployedInternally()) { Optional<Run> run = controller.jobController().last(application.id(), jobType); return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 1acf9bef363..5fd197eb785 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -40,6 +40,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -276,15 +277,17 @@ public class InternalStepRunner implements StepRunner { private boolean nodesConverged(ApplicationId id, JobType type, Version target, DualLogger logger) { List<Node> nodes = controller.configServer().nodeRepository().list(type.zone(controller.system()), id, ImmutableSet.of(active, reserved)); - for (Node node : nodes) - logger.log(String.format("%70s: %-16s%-25s%-32s%s", + List<String> statuses = nodes.stream() + .map(node -> String.format("%70s: %-16s%-25s%-32s%s", node.hostname(), node.serviceState(), node.wantedVersion() + (node.currentVersion().equals(node.wantedVersion()) ? "" : " <-- " + node.currentVersion()), node.restartGeneration() == node.wantedRestartGeneration() ? "" : "restart pending (" + node.wantedRestartGeneration() + " <-- " + node.restartGeneration() + ")", node.rebootGeneration() == node.wantedRebootGeneration() ? "" - : "reboot pending (" + node.wantedRebootGeneration() + " <-- " + node.rebootGeneration() + ")")); + : "reboot pending (" + node.wantedRebootGeneration() + " <-- " + node.rebootGeneration() + ")")) + .collect(Collectors.toList()); + logger.log(statuses); return nodes.stream().allMatch(node -> node.currentVersion().equals(target) && node.restartGeneration() == node.wantedRestartGeneration() @@ -298,13 +301,16 @@ public class InternalStepRunner implements StepRunner { return false; } logger.log("Wanted config generation is " + convergence.get().wantedGeneration()); - for (ServiceConvergence.Status serviceStatus : convergence.get().services()) - if (serviceStatus.currentGeneration() != convergence.get().wantedGeneration()) - logger.log(String.format("%70s: %11s on port %4d has %s", - serviceStatus.host().value(), - serviceStatus.type(), - serviceStatus.port(), - serviceStatus.currentGeneration() == -1 ? "(unknown)" : Long.toString(serviceStatus.currentGeneration()))); + List<String> statuses = convergence.get().services().stream() + .filter(serviceStatus -> serviceStatus.currentGeneration() != convergence.get().wantedGeneration()) + .map(serviceStatus -> String.format("%70s: %11s on port %4d has %s", + serviceStatus.host().value(), + serviceStatus.type(), + serviceStatus.port(), + serviceStatus.currentGeneration() == -1 ? "(unknown)" : Long.toString(serviceStatus.currentGeneration()))) + .collect(Collectors.toList()); + logger.log(statuses); + return convergence.get().converged(); } @@ -432,7 +438,7 @@ public class InternalStepRunner implements StepRunner { private DeploymentJobs.JobReport report(Run run) { return new DeploymentJobs.JobReport(run.id().application(), run.id().type(), - 1, + controller.applications().require(run.id().application()).deploymentJobs().projectId().orElse(1), run.id().number(), Optional.empty(), run.hasFailed() ? Optional.of(DeploymentJobs.JobError.unknown) : Optional.empty()); @@ -572,16 +578,18 @@ public class InternalStepRunner implements StepRunner { private final RunId id; private final Step step; - private final String prefix; private DualLogger(RunId id, Step step) { this.id = id; this.step = step; - this.prefix = id + " at " + step + ": "; } - private void log(String message) { - log(DEBUG, message); + private void log(String... messages) { + log(Arrays.asList(messages)); + } + + private void log(List<String> messages) { + controller.jobController().log(id, step, DEBUG, messages); } private void log(Level level, String message) { @@ -589,7 +597,7 @@ public class InternalStepRunner implements StepRunner { } private void log(Level level, String message, Throwable thrown) { - logger.log(level, message, thrown); + logger.log(level, id + " at " + step + ": " + message, thrown); if (thrown != null) { ByteArrayOutputStream traceBuffer = new ByteArrayOutputStream(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 453447830bb..a75c3d2f8f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -36,6 +36,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -102,15 +103,22 @@ public class JobController { } } - /** Stores the given log record for the given run and step. */ - public void log(RunId id, Step step, Level level, String message) { + /** Stores the given log records for the given run and step. */ + public void log(RunId id, Step step, Level level, List<String> messages) { locked(id, __ -> { - LogEntry entry = new LogEntry(0, controller.clock().millis(), LogEntry.typeOf(level), message); - logs.append(id.application(), id.type(), step, Collections.singletonList(entry)); + List<LogEntry> entries = messages.stream() + .map(message -> new LogEntry(0, controller.clock().millis(), LogEntry.typeOf(level), message)) + .collect(Collectors.toList()); + logs.append(id.application(), id.type(), step, entries); return __; }); } + /** Stores the given log record for the given run and step. */ + public void log(RunId id, Step step, Level level, String message) { + log(id, step, level, Collections.singletonList(message)); + } + /** Fetches any new test log entries, and records the id of the last of these, for continuation. */ public void updateTestLog(RunId id) { locked(id, run -> { @@ -133,7 +141,7 @@ public class JobController { /** Returns a list of all application which have registered. */ public List<ApplicationId> applications() { return copyOf(controller.applications().asList().stream() - .filter(application -> application.deploymentJobs().builtInternally()) + .filter(application -> application.deploymentJobs().deployedInternally()) .map(Application::id) .iterator()); } @@ -212,11 +220,11 @@ public class JobController { /** * Accepts and stores a new application package and test jar pair under a generated application version key. */ - public ApplicationVersion submit(ApplicationId id, SourceRevision revision, + public ApplicationVersion submit(ApplicationId id, SourceRevision revision, long projectId, byte[] packageBytes, byte[] testPackageBytes) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { - if ( ! application.get().deploymentJobs().builtInternally()) { + if ( ! application.get().deploymentJobs().deployedInternally()) { // Copy all current packages to the new application store application.get().deployments().values().stream() .map(Deployment::applicationVersion) @@ -239,7 +247,7 @@ public class JobController { controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), new ApplicationPackage(packageBytes)); - notifyOfNewSubmission(id, revision, run); + notifyOfNewSubmission(id, projectId, revision, run); }); return version.get(); } @@ -247,7 +255,7 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions) { controller.applications().lockIfPresent(id, application -> { - if ( ! application.get().deploymentJobs().builtInternally()) + if ( ! application.get().deploymentJobs().deployedInternally()) throw new IllegalArgumentException(id + " is not built here!"); locked(id, type, __ -> { @@ -330,10 +338,10 @@ public class JobController { } // TODO jvenstad: Find a more appropriate way of doing this when this is the only build service. - private void notifyOfNewSubmission(ApplicationId id, SourceRevision revision, long number) { + private void notifyOfNewSubmission(ApplicationId id, long projectId, SourceRevision revision, long number) { DeploymentJobs.JobReport report = new DeploymentJobs.JobReport(id, JobType.component, - 1, + projectId, number, Optional.of(revision), Optional.empty()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index b2b69a81dbc..219517cfd30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; -import java.io.UncheckedIOException; import java.time.Duration; import java.util.NoSuchElementException; import java.util.Optional; @@ -44,7 +43,6 @@ public class ApplicationOwnershipConfirmer extends Maintainer { /** File an ownership issue with the owners of all applications we know about. */ private void confirmApplicationOwnerships() { ApplicationList.from(controller().applications().asList()) - .notPullRequest() .withProjectId() .hasProductionDeployment() .asList() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index aa0ad8b63e0..4a49d5df5ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -89,7 +88,7 @@ public class ClusterInfoMaintainer extends Maintainer { @Override protected void maintain() { - for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { + for (Application application : controller().applications().asList()) { for (Deployment deployment : application.deployments().values()) { DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone()); try { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index a046ed87a05..44f29eb6113 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,17 +3,26 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; import java.time.Duration; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.logging.Level; /** * Fetch utilization metrics and update applications with this data. @@ -23,10 +32,12 @@ import java.util.Map; public class ClusterUtilizationMaintainer extends Maintainer { private final Controller controller; + private final List<String> baseUris; - public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) { + public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl, ApiAuthorityConfig apiAuthorityConfig) { super(controller, duration, jobControl); this.controller = controller; + this.baseUris = apiAuthorityConfig.authorities(); } private Map<ClusterSpec.Id, ClusterUtilization> getUpdatedClusterUtilizations(ApplicationId app, ZoneId zone) { @@ -44,15 +55,45 @@ public class ClusterUtilizationMaintainer extends Maintainer { @Override protected void maintain() { - for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { - for (Deployment deployment : application.deployments().values()) { + try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { + String uri = baseUris.get(0) + "metricforwarding/v1/clusterutilization"; // For now, we only feed to one controller + Slime slime = getMetricSlime(); + ByteArrayEntity entity = new ByteArrayEntity(SlimeUtils.toJsonBytes(slime)); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(entity); + httpClient.execute(httpPost); + } catch (Exception e) { + log.log(Level.WARNING, "Failed to update cluster utilization metrics", e); + } - Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone()); + } - controller().applications().lockIfPresent(application.id(), lockedApplication -> - controller().applications().store(lockedApplication.withClusterUtilization(deployment.zone(), clusterUtilization))); + private Slime getMetricSlime() { + Slime slime = new Slime(); + Cursor cursor = slime.setArray(); + for (Application application : controller().applications().asList()) { + Cursor applicationCursor = cursor.addObject(); + applicationCursor.setString("applicationId", application.id().serializedForm()); + Cursor deploymentArray = applicationCursor.setArray("deployments"); + for (Deployment deployment : application.deployments().values()) { + Cursor deploymentEntry = deploymentArray.addObject(); + deploymentEntry.setString("zoneId", deployment.zone().value()); + Cursor clusterArray = deploymentEntry.setArray("clusterUtil"); + Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone()); + fillClusterUtilization(clusterArray, clusterUtilization); } } + return slime; } + private void fillClusterUtilization(Cursor cursor, Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization) { + for (Map.Entry<ClusterSpec.Id, ClusterUtilization> entry : clusterUtilization.entrySet()) { + Cursor clusterUtilCursor = cursor.addObject(); + clusterUtilCursor.setString("clusterSpecId", entry.getKey().value()); + clusterUtilCursor.setDouble("cpu", entry.getValue().getCpu()); + clusterUtilCursor.setDouble("memory", entry.getValue().getMemory()); + clusterUtilCursor.setDouble("disk", entry.getValue().getDisk()); + clusterUtilCursor.setDouble("diskBusy", entry.getValue().getDiskBusy()); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 33555c08a43..5d26ce16885 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -65,8 +65,8 @@ public class ControllerMaintenance extends AbstractComponent { upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofSeconds(30), jobControl); clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient); - clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); - deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); + clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl, apiAuthorityConfig); + deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl, apiAuthorityConfig); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 11e0ada6c36..1745e013a40 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; -import java.io.UncheckedIOException; import java.time.Duration; import java.util.Collection; import java.util.List; @@ -57,7 +56,6 @@ public class DeploymentIssueReporter extends Maintainer { private List<Application> applications() { return ApplicationList.from(controller().applications().asList()) .withProjectId() - .notPullRequest() .asList(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 0b56916a0de..67fb224f1ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -2,16 +2,23 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.RotationStatus; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.yolean.Exceptions; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import java.io.IOException; import java.time.Duration; import java.util.Collections; import java.util.List; @@ -38,51 +45,43 @@ public class DeploymentMetricsMaintainer extends Maintainer { private static final int applicationsToUpdateInParallel = 10; private final ApplicationController applications; + private final List<String> baseUris; - public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) { + public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl, ApiAuthorityConfig apiAuthorityConfig) { super(controller, duration, jobControl); this.applications = controller.applications(); + baseUris = apiAuthorityConfig.authorities(); } @Override protected void maintain() { AtomicInteger failures = new AtomicInteger(0); AtomicReference<Exception> lastException = new AtomicReference<>(null); - List<Application> applicationList = ApplicationList.from(applications.asList()).notPullRequest().asList(); + List<Application> applicationList = applications.asList(); // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel); + Slime slime = new Slime(); + Cursor cursor = slime.setArray(); pool.submit(() -> { applicationList.parallelStream().forEach(application -> { - try { - applications.lockIfPresent(application.id(), locked -> - applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id())))); - - applications.lockIfPresent(application.id(), locked -> - applications.store(locked.withRotationStatus(rotationStatus(application)))); - - for (Deployment deployment : application.deployments().values()) { - MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService() - .getDeploymentMetrics(application.id(), deployment.zone()); - DeploymentMetrics newMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(), - deploymentMetrics.writesPerSecond(), - deploymentMetrics.documentCount(), - deploymentMetrics.queryLatencyMillis(), - deploymentMetrics.writeLatencyMillis()); - - applications.lockIfPresent(application.id(), locked -> - applications.store(locked.with(deployment.zone(), newMetrics) - .recordActivityAt(controller().clock().instant(), deployment.zone()))); - } - } catch (Exception e) { - failures.incrementAndGet(); - lastException.set(e); + Cursor applicationCursor = cursor.addObject(); + applicationCursor.setString("applicationId", application.id().serializedForm()); + Cursor applicationMetrics = applicationCursor.setObject("applicationMetrics"); + fillApplicationMetrics(applicationMetrics, application); + Cursor rotationStatus = applicationCursor.setArray("rotationStatus"); + fillRotationStatus(rotationStatus, application); + Cursor deploymentArray = applicationCursor.setArray("deploymentMetrics"); + for (Deployment deployment : application.deployments().values()) { + Cursor deploymentEntry = deploymentArray.addObject(); + fillDeploymentMetrics(deploymentEntry, application, deployment); } }); }); pool.shutdown(); try { pool.awaitTermination(30, TimeUnit.MINUTES); + feedMetrics(slime); if (lastException.get() != null) { log.log(Level.WARNING, String.format("Failed to query metrics service for %d/%d applications. Last error: %s. Retrying in %s", failures.get(), @@ -92,6 +91,8 @@ public class DeploymentMetricsMaintainer extends Maintainer { } } catch (InterruptedException e) { throw new RuntimeException(e); + } catch (IOException e) { + log.log(Level.WARNING, "Unable to feed metrics to API", e); } } @@ -107,6 +108,40 @@ public class DeploymentMetricsMaintainer extends Maintainer { .orElseGet(Collections::emptyMap); } + private void fillApplicationMetrics(Cursor applicationCursor, Application application) { + MetricsService.ApplicationMetrics metrics = controller().metricsService().getApplicationMetrics(application.id()); + applicationCursor.setDouble("queryServiceQuality", metrics.queryServiceQuality()); + applicationCursor.setDouble("writeServiceQuality", metrics.writeServiceQuality()); + } + + private void fillRotationStatus(Cursor rotationStatusCursor, Application application) { + Map<HostName, RotationStatus> rotationStatus = rotationStatus(application); + for (Map.Entry<HostName, RotationStatus> entry : rotationStatus.entrySet()) { + Cursor rotationStatusEntry = rotationStatusCursor.addObject(); + rotationStatusEntry.setString("hostname", entry.getKey().value()); + rotationStatusEntry.setString("rotationStatus", entry.getValue().toString()); + } + } + + private void fillDeploymentMetrics(Cursor deploymentCursor, Application application, Deployment deployment) { + MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService() + .getDeploymentMetrics(application.id(), deployment.zone()); + deploymentCursor.setString("zoneId", deployment.zone().value()); + deploymentCursor.setDouble("queriesPerSecond", deploymentMetrics.queriesPerSecond()); + deploymentCursor.setDouble("writesPerSecond", deploymentMetrics.writesPerSecond()); + deploymentCursor.setDouble("documentCount", deploymentMetrics.documentCount()); + deploymentCursor.setDouble("queryLatencyMillis", deploymentMetrics.queryLatencyMillis()); + deploymentCursor.setDouble("writeLatencyMillis", deploymentMetrics.writeLatencyMillis()); + } + + private void feedMetrics(Slime slime) throws IOException { + String uri = baseUris.get(0) + "/metricforwarding/v1/deploymentmetrics/"; // For now, we only feed to one controller + CloseableHttpClient httpClient = HttpClientBuilder.create().build(); + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new ByteArrayEntity(SlimeUtils.toJsonBytes(slime))); + httpClient.execute(httpPost); + } + private static RotationStatus from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) { switch (status) { case IN: return RotationStatus.in; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 258a72c8d01..305241a7d0e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -117,7 +117,6 @@ public class MetricsReporter extends Maintainer { private void reportDeploymentMetrics() { ApplicationList applications = ApplicationList.from(controller().applications().asList()) - .notPullRequest() .hasProductionDeployment(); metric.set(deploymentFailMetric, deploymentFailRatio(applications) * 100, metric.createContext(Collections.emptyMap())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index 63361e5dcc4..f87a0d625c0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; import java.time.Duration; @@ -20,8 +19,7 @@ public class OutstandingChangeDeployer extends Maintainer { @Override protected void maintain() { - ApplicationList applications = ApplicationList.from(controller().applications().asList()).notPullRequest(); - for (Application application : applications.asList()) { + for (Application application : controller().applications().asList()) { if ( ! application.change().isPresent() && application.outstandingChange().isPresent()) { controller().applications().deploymentTrigger().triggerChange(application.id(), application.outstandingChange()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 676fe808bfe..06712ad7862 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -92,7 +92,6 @@ public class Upgrader extends Maintainer { private ApplicationList applications() { return ApplicationList.from(controller().applications().asList()); } private void upgrade(ApplicationList applications, Version version) { - applications = applications.notPullRequest(); // Pull requests are deployed as separate applications to test then deleted; No need to upgrade applications = applications.hasProductionDeployment(); applications = applications.onLowerVersionThan(version); applications = applications.allowMajorVersion(version.getMajor()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 1452e3aa61d..7b615249a65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -235,7 +235,7 @@ public class ApplicationSerializer { deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId)); jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField)); deploymentJobs.issueId().ifPresent(jiraIssueId -> cursor.setString(issueIdField, jiraIssueId.value())); - cursor.setBool(builtInternallyField, deploymentJobs.builtInternally()); + cursor.setBool(builtInternallyField, deploymentJobs.deployedInternally()); } private void jobStatusToSlime(Collection<JobStatus> jobStatuses, Cursor jobStatusArray) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index 3df4451f900..2833bc8614e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -24,7 +24,7 @@ import java.util.stream.Collectors; */ public class BufferedLogStore { - static final int chunkSize = 1 << 17; + static final int chunkSize = 1 << 13; private final CuratorDb buffer; private final RunDataStore store; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 9ee477e5ab2..631aaa5a909 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -410,7 +410,8 @@ public class CuratorDb { public LongStream getLogChunkIds(ApplicationId id, JobType type) { return curator.getChildren(runsPath(id, type).append("logs")).stream() - .mapToLong(Long::parseLong); + .mapToLong(Long::parseLong) + .sorted(); } // -------------- Provisioning (called by internal code) ------------------ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 57133986654..bcf0ff70c64 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -24,7 +24,8 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.api.OktaAccessToken; +import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.AlreadyExistsException; import com.yahoo.vespa.hosted.controller.Application; @@ -46,7 +47,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; @@ -65,6 +65,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; @@ -109,7 +110,7 @@ import java.util.stream.Collectors; public class ApplicationApiHandler extends LoggingRequestHandler { private final Controller controller; - private final AthenzClientFactory athenzClientFactory; + private final ZmsClientFacade zmsClient; @Inject public ApplicationApiHandler(LoggingRequestHandler.Context parentCtx, @@ -117,7 +118,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { AthenzClientFactory athenzClientFactory) { super(parentCtx); this.controller = controller; - this.athenzClientFactory = athenzClientFactory; + this.zmsClient = new ZmsClientFacade(athenzClientFactory.createZmsClient(), athenzClientFactory.getControllerIdentity()); } @Override @@ -175,6 +176,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); @@ -211,6 +213,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); @@ -395,6 +398,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .steps(application.deploymentSpec()) .sortedJobs(application.deploymentJobs().jobStatus().values()); + object.setBool("deployedInternally", application.deploymentJobs().deployedInternally()); Cursor deploymentsArray = object.setArray("deploymentJobs"); for (JobStatus job : jobStatus) { Cursor jobObject = deploymentsArray.addObject(); @@ -636,11 +640,21 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } + private HttpResponse suspended(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + ZoneId.from(environment, region)); + boolean suspended = controller.applications().isSuspended(deploymentId); + Slime slime = new Slime(); + Cursor response = slime.setObject(); + response.setBool("suspended", suspended); + return new SlimeJsonResponse(slime); + } + private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - controller.zoneRegistry().getConfigServerUris(ZoneId.from(environment, region)), + controller.zoneRegistry().getConfigServerApiUris(ZoneId.from(environment, region)), request.getUri()); response.setResponse(applicationView); return response; @@ -650,7 +664,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath); ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - controller.zoneRegistry().getConfigServerUris(ZoneId.from(environment, region)), + controller.zoneRegistry().getConfigServerApiUris(ZoneId.from(environment, region)), request.getUri()); response.setResponse(result, serviceName, restPath); return response; @@ -675,7 +689,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if ( ! tenant.isPresent()) return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist"); Inspector requestData = toSlime(request.getData()).get(); - NToken token = requireNToken(request, "Could not update " + tenantName); + OktaAccessToken token = requireOktaAccessToken(request, "Could not update " + tenantName); controller.tenants().lockOrThrow(tenant.get().name(), lockedTenant -> { lockedTenant = lockedTenant.with(new Property(mandatory("property", requestData).asString())); @@ -702,17 +716,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { new Property(mandatory("property", requestData).asString()), optional("propertyId", requestData).map(PropertyId::new)); throwIfNotAthenzDomainAdmin(tenant.domain(), request); - controller.tenants().create(tenant, requireNToken(request, "Could not create " + tenantName)); + controller.tenants().create(tenant, requireOktaAccessToken(request, "Could not create " + tenantName)); return tenant(tenant, request, true); } private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { Application application; try { - application = controller.applications().createApplication(ApplicationId.from(tenantName, applicationName, "default"), getUserPrincipal(request).getNToken()); + application = controller.applications().createApplication(ApplicationId.from(tenantName, applicationName, "default"), getOktaAccessToken(request)); } - catch (ZmsException e) { // TODO: Push conversion down - if (e.getCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) + catch (ZmsClientException e) { // TODO: Push conversion down + if (e.getErrorCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) throw new ForbiddenException("Not authorized to create application", e); else throw e; @@ -828,7 +842,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (tenant.get() instanceof AthenzTenant) { controller.tenants().deleteTenant((AthenzTenant) tenant.get(), - requireNToken(request, "Could not delete " + tenantName)); + requireOktaAccessToken(request, "Could not delete " + tenantName)); } else if (tenant.get() instanceof UserTenant) { controller.tenants().deleteTenant((UserTenant) tenant.get()); } else { @@ -842,7 +856,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); - controller.applications().deleteApplication(id, getUserPrincipal(request).getNToken()); + controller.applications().deleteApplication(id, getOktaAccessToken(request)); return new EmptyJsonResponse(); // TODO: Replicates current behavior but should return a message response instead } @@ -1019,8 +1033,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void throwIfNotAthenzDomainAdmin(AthenzDomain tenantDomain, HttpRequest request) { AthenzIdentity identity = getUserPrincipal(request).getIdentity(); - boolean isDomainAdmin = athenzClientFactory.createZmsClientWithServicePrincipal() - .isDomainAdmin(identity, tenantDomain); + boolean isDomainAdmin = zmsClient.isDomainAdmin(identity, tenantDomain); if ( ! isDomainAdmin) { throw new ForbiddenException( String.format("The user '%s' is not admin in Athenz domain '%s'", identity.getFullName(), tenantDomain.getName())); @@ -1212,9 +1225,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throw new IllegalArgumentException("Unknown tenant type: " + tenant.getClass().getSimpleName()); } - private static NToken requireNToken(HttpRequest request, String message) { - return getUserPrincipal(request).getNToken().orElseThrow(() -> new IllegalArgumentException( - message + ": No NToken provided")); + private static OktaAccessToken requireOktaAccessToken(HttpRequest request, String message) { + return getOktaAccessToken(request) + .orElseThrow(() -> new IllegalArgumentException(message + ": No Okta Access Token provided")); + } + + private static Optional<OktaAccessToken> getOktaAccessToken(HttpRequest request) { + return Optional.ofNullable(request.getHeader(OktaAccessToken.HTTP_HEADER_NAME)) + .map(OktaAccessToken::new); } private static ApplicationId appIdFromPath(Path path) { @@ -1235,6 +1253,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); SourceRevision sourceRevision = toSourceRevision(submitOptions).orElseThrow(() -> new IllegalArgumentException("Must specify 'repository', 'branch', and 'commit'")); + long projectId = Math.max(1, submitOptions.field("projectId").asLong()); ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); if ( ! applicationPackage.deploymentSpec().athenzDomain().isPresent()) @@ -1242,8 +1261,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { verifyApplicationIdentityConfiguration(tenant, Optional.of(applicationPackage)); return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, - sourceRevision, - applicationPackage.zippedContent(), - dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); + sourceRevision, + projectId, + applicationPackage.zippedContent(), + dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 05c7bd5e11b..2e67f4b2efc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.net.URI; @@ -394,9 +395,12 @@ class JobControllerApiHandlerHelper { * @return Response with the new application version */ static HttpResponse submitResponse(JobController jobController, String tenant, String application, - SourceRevision sourceRevision, byte[] appPackage, byte[] testPackage) { + SourceRevision sourceRevision, long projectId, byte[] appPackage, byte[] testPackage) { ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, "default"), - sourceRevision, appPackage, testPackage); + sourceRevision, + projectId, + appPackage, + testPackage); Slime slime = new Slime(); Cursor responseObject = slime.setObject(); @@ -404,5 +408,19 @@ class JobControllerApiHandlerHelper { return new SlimeJsonResponse(slime); } + /** Aborts any job of the given type. */ + static HttpResponse abortJobResponse(JobController jobs, ApplicationId id, JobType type) { + Slime slime = new Slime(); + Cursor responseObject = slime.setObject(); + Optional<Run> run = jobs.last(id, type).flatMap(last -> jobs.active(last.id())); + if (run.isPresent()) { + jobs.abort(run.get().id()); + responseObject.setString("message", "Aborting " + run); + } + else + responseObject.setString("message", "Nothing to abort."); + return new SlimeJsonResponse(slime); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 33d53b0becf..59847437339 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -9,16 +9,17 @@ import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.cors.CorsFilterConfig; import com.yahoo.jdisc.http.filter.security.cors.CorsRequestFilterBase; import com.yahoo.log.LogLevel; +import com.yahoo.restapi.Path; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; -import com.yahoo.restapi.Path; +import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.UserTenant; @@ -40,7 +41,7 @@ import static com.yahoo.jdisc.http.HttpRequest.Method.HEAD; import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS; import static com.yahoo.jdisc.http.HttpRequest.Method.POST; import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; -import static com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities.SCREWDRIVER_DOMAIN; +import static com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities.SCREWDRIVER_DOMAIN; /** * A security filter protects all controller apis. @@ -55,7 +56,7 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { private static final Logger log = Logger.getLogger(ControllerAuthorizationFilter.class.getName()); - private final AthenzClientFactory clientFactory; + private final ZmsClientFacade zmsClient; private final TenantController tenantController; @Inject @@ -63,7 +64,7 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { Controller controller, CorsFilterConfig corsConfig) { super(corsConfig); - this.clientFactory = clientFactory; + this.zmsClient = new ZmsClientFacade(clientFactory.createZmsClient(), clientFactory.getControllerIdentity()); this.tenantController = controller.tenants(); } @@ -71,7 +72,7 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { TenantController tenantController, Set<String> allowedUrls) { super(allowedUrls); - this.clientFactory = clientFactory; + this.zmsClient = new ZmsClientFacade(clientFactory.createZmsClient(), clientFactory.getControllerIdentity());; this.tenantController = tenantController; } @@ -128,20 +129,21 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { private static boolean isTenantAdminOperation(Path path, Method method) { if (isHostedOperatorOperation(path, method)) return false; return path.matches("/application/v4/tenant/{tenant}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"); + path.matches("/application/v4/tenant/{tenant}/application/{application}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{job}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"); } private static boolean isTenantPipelineOperation(Path path, Method method) { if (isTenantAdminOperation(path, method)) return false; return path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/submit") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/promote") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/prod/{*}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/test/{*}") || - path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/staging/{*}"); + path.matches("/application/v4/tenant/{tenant}/application/{application}/submit") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/promote") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/prod/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/test/{*}") || + path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/staging/{*}"); } private void verifyIsHostedOperator(AthenzPrincipal principal) { @@ -151,8 +153,7 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { } private boolean isHostedOperator(AthenzIdentity identity) { - return clientFactory.createZmsClientWithServicePrincipal() - .hasHostedOperatorAccess(identity); + return zmsClient.hasHostedOperatorAccess(identity); } private void verifyIsTenantAdmin(AthenzPrincipal principal, TenantName name) { @@ -166,8 +167,7 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { private boolean isTenantAdmin(AthenzIdentity identity, Tenant tenant) { if (tenant instanceof AthenzTenant) { - return clientFactory.createZmsClientWithServicePrincipal() - .hasTenantAdminAccess(identity, ((AthenzTenant) tenant).domain()); + return zmsClient.hasTenantAdminAccess(identity, ((AthenzTenant) tenant).domain()); } else if (tenant instanceof UserTenant) { if (!(identity instanceof AthenzUser)) { return false; @@ -210,13 +210,13 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { private boolean hasDeployerAccess(AthenzIdentity identity, AthenzDomain tenantDomain, ApplicationName application) { try { - return clientFactory.createZmsClientWithServicePrincipal() + return zmsClient .hasApplicationAccess( identity, ApplicationAction.deploy, tenantDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(application.value())); - } catch (ZmsException e) { + } catch (ZmsClientException e) { throw new InternalServerErrorException("Failed to authorize operation: (" + e.getMessage() + ")", e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/metrics/MetricForwardingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/metrics/MetricForwardingApiHandler.java new file mode 100644 index 00000000000..91c847d10f1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/metrics/MetricForwardingApiHandler.java @@ -0,0 +1,153 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.metrics; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.io.IOUtils; +import com.yahoo.restapi.Path; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.StringResponse; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * This implements the metricforwarding/v1 API which allows feeding + * MetricsService data. + * @author olaa + */ +public class MetricForwardingApiHandler extends LoggingRequestHandler { + + private final Controller controller; + + public MetricForwardingApiHandler(Context ctx, Controller controller) { + super(ctx); + this.controller = controller; + } + + @Override + public HttpResponse handle(HttpRequest request) { + switch (request.getMethod()) { + case POST: + return post(request); + default: + return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + } + } + + private HttpResponse post(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/metricforwarding/v1/clusterutilization")) return updateClusterUtilization(request); + if (path.matches("/metricforwarding/v1/deploymentmetrics")) return updateDeploymentMetrics(request); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse updateClusterUtilization(HttpRequest request) { + try { + Slime slime = SlimeUtils.jsonToSlime(IOUtils.readBytes(request.getData(), 1000 * 1000)); + Inspector inspector = slime.get(); + inspector.traverse((ArrayTraverser) (index, entry) -> { + ApplicationId applicationId = ApplicationId.fromSerializedForm(entry.field("applicationId").asString()); + entry.field("deployments").traverse((ArrayTraverser) (i, deployment) -> { + ZoneId zoneId = ZoneId.from(deployment.field("zoneId").asString()); + Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getClusterUtilizationsFromInspector(deployment.field("clusterUtil")); + controller.applications().lockIfPresent(applicationId, lockedApplication -> + controller.applications().store(lockedApplication.withClusterUtilization(zoneId, clusterUtilization))); + }); + }); + } catch (IOException e) { + ErrorResponse.badRequest("Unable to parse request for metrics - " + e.getMessage()); + } + return new StringResponse("Added cluster utilization metrics"); + } + + private HttpResponse updateDeploymentMetrics(HttpRequest request) { + try { + Slime slime = SlimeUtils.jsonToSlime(IOUtils.readBytes(request.getData(), 1000 * 1000)); + Inspector inspector = slime.get(); + inspector.traverse((ArrayTraverser) (index, applicationEntry) -> { + ApplicationId applicationId = ApplicationId.fromSerializedForm(applicationEntry.field("applicationId").asString()); + MetricsService.ApplicationMetrics applicationMetrics = getApplicationMetricsFromInspector(applicationEntry); + ApplicationController applications = controller.applications(); + applications.lockIfPresent(applicationId, lockedApplication -> + applications.store(lockedApplication.with(applicationMetrics))); + + Map<HostName, RotationStatus> rotationStatusMap = getRotationStatusFromInspector(applicationEntry); + applications.lockIfPresent(applicationId, lockedApplication -> + applications.store(lockedApplication.withRotationStatus(rotationStatusMap))); + + for (Map.Entry<ZoneId, DeploymentMetrics> entry : getDeploymentMetricsFromInspector(applicationEntry).entrySet()) { + applications.lockIfPresent(applicationId, lockedApplication -> + applications.store(lockedApplication.with(entry.getKey(), entry.getValue()) + .recordActivityAt(controller.clock().instant(), entry.getKey()))); + } + }); + } catch (IOException e) { + ErrorResponse.badRequest("Unable to parse request for metrics - " + e.getMessage()); + } + return new StringResponse("Added deployment metrics"); + } + + private Map<ClusterSpec.Id, ClusterUtilization> getClusterUtilizationsFromInspector(Inspector inspector) { + Map<ClusterSpec.Id, ClusterUtilization> clusterUtilizationMap = new HashMap<>(); + inspector.traverse((ArrayTraverser) (index, entry) -> { + ClusterSpec.Id id = ClusterSpec.Id.from(entry.field("clusterSpecId").asString()); + double memory = entry.field("memory").asDouble(); + double cpu = entry.field("cpu").asDouble(); + double disk = entry.field("disk").asDouble(); + double diskBusy = entry.field("diskBusy").asDouble(); + clusterUtilizationMap.put(id, new ClusterUtilization(memory, cpu, disk, diskBusy)); + }); + return clusterUtilizationMap; + } + + private Map<ZoneId, DeploymentMetrics> getDeploymentMetricsFromInspector(Inspector inspector){ + Map<ZoneId, DeploymentMetrics> deploymentMetricsMap = new HashMap<>(); + inspector = inspector.field("deploymentMetrics"); + inspector.traverse((ArrayTraverser) (index, entry) -> { + ZoneId zoneId = ZoneId.from(entry.field("zoneId").asString()); + double queriesPerSecond = entry.field("queriesPerSecond").asDouble(); + double writesPerSecond = entry.field("writesPerSecond").asDouble(); + double documentCount = entry.field("documentCount").asDouble(); + double queryLatencyMillis = entry.field("queryLatencyMillis").asDouble(); + double writeLatencyMillis = entry.field("writeLatencyMillis").asDouble(); + DeploymentMetrics metrics = new DeploymentMetrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, writeLatencyMillis); + deploymentMetricsMap.put(zoneId, metrics); + }); + return deploymentMetricsMap; + } + + private MetricsService.ApplicationMetrics getApplicationMetricsFromInspector(Inspector inspector){ + inspector = inspector.field("applicationMetrics"); + double queryServiceQuality = inspector.field("queryServiceQuality").asDouble(); + double writeServiceQuality = inspector.field("writeServiceQuality").asDouble(); + return new MetricsService.ApplicationMetrics(queryServiceQuality, writeServiceQuality); + } + + private Map<HostName, RotationStatus> getRotationStatusFromInspector(Inspector inspector) { + Map<HostName, RotationStatus> rotationStatusMap = new HashMap<>(); + inspector = inspector.field("rotationStatus"); + inspector.traverse((ArrayTraverser) (index, entry) -> { + HostName hostName = HostName.from(entry.field("hostname").asString()); + RotationStatus rotationStatus = RotationStatus.valueOf(entry.field("rotationStatus").asString()); + rotationStatusMap.put(hostName, rotationStatus); + }); + return rotationStatusMap; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index 74caa4dcb47..6633eafc509 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -6,15 +6,14 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; -import com.yahoo.restapi.Path; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.io.InputStream; @@ -64,11 +63,9 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { private HttpResponse get(HttpRequest request) { Path path = new Path(request.getUri().getPath()); - if (path.matches("/screwdriver/v1/release/vespa")) { - return vespaVersion(); - } - if (path.matches("/screwdriver/v1/jobsToRun")) + if (path.matches("/screwdriver/v1/jobsToRun")) { return buildJobs(controller.applications().deploymentTrigger().jobsToRun()); + } return notFound(request); } @@ -97,19 +94,6 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse vespaVersion() { - VespaVersion version = controller.versionStatus().version(controller.systemVersion()); - if (version == null) - return ErrorResponse.notFoundError("Information about the current system version is not available at this time"); - - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - cursor.setString("version", version.versionNumber().toString()); - cursor.setString("sha", version.releaseCommit()); - cursor.setLong("date", version.committedAt().toEpochMilli()); - return new SlimeJsonResponse(slime); - } - private HttpResponse buildJobs(Map<JobType, ? extends List<? extends BuildService.BuildJob>> jobLists) { Slime slime = new Slime(); Cursor jobTypesObject = slime.setObject(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index 332c440d18e..09f8de40378 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -65,7 +65,7 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple // Key store containing key pair from secret store factory.setKeyStore(KeyStoreBuilder.withType(KeyStoreType.JKS) - .withKeyEntry(getClass().getSimpleName(), privateKey(), certificate()) + .withKeyEntry(getClass().getSimpleName(), privateKey(), certificates()) .build()); factory.setKeyStorePassword(""); @@ -77,8 +77,11 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple return KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(config.privateKeySecret())); } - /** Get certificate from secret store */ - private List<X509Certificate> certificate() { + /** + * Get certificate from secret store. If certificate secret contains multiple certificates, e.g. intermediate + * certificates, the entire chain will be read + */ + private List<X509Certificate> certificates() { return X509CertificateUtils.certificateListFromPem(secretStore.getSecret(config.certificateSecret())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 8b70a49576b..e367de35d46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -188,7 +188,6 @@ public class VersionStatus { } ApplicationList applicationList = ApplicationList.from(applications) - .notPullRequest() .hasProductionDeployment(); for (Application application : applicationList.asList()) { // Note that each version deployed on this application in production exists diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 06896412652..ffbf24be12a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -51,8 +51,7 @@ public class VespaVersion implements Comparable<VespaVersion> { .notFailing(); ApplicationList failingOnThis = ApplicationList.from(statistics.failing(), controller.applications()); ApplicationList all = ApplicationList.from(controller.applications().asList()) - .hasDeployment() - .notPullRequest(); + .hasDeployment(); // 'broken' if any Canary fails if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty()) diff --git a/controller-server/src/main/resources/configdefinitions/athenz.def b/controller-server/src/main/resources/configdefinitions/athenz.def index f8d65c25e47..dc1a2337aaf 100644 --- a/controller-server/src/main/resources/configdefinitions/athenz.def +++ b/controller-server/src/main/resources/configdefinitions/athenz.def @@ -1,44 +1,15 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=vespa.hosted.controller.athenz.config -# Principal header name -principalHeaderName string default="Athenz-Principal-Auth" -# TODO Remove once migrated to Okta - # URL to ZMS API endpoint zmsUrl string -# URL to legacy ZMS API endpoint -legacyZmsUrl string -# TODO Remove once migrated to Okta - # URL to ZTS API endpoint ztsUrl string # Athenz domain for controller identity. The domain is also used for Athenz tenancy integration. domain string -# Name of the internal user authentication passthru attribute -userAuthenticationPassThruAttribute string -# TODO Remove once migrated to Okta - -# Path to Athenz CA JKS trust store -athenzCaTrustStore string - -# Certificate DNS domain -certDnsDomain string - # Athenz service name for controller identity service.name string -# Athenz service public key id -service.publicKeyId string - -# Version of Athenz service private key -service.privateKeyVersion int - -# Name of Athenz service private key secret -service.privateKeySecretName string - -# Expiry of service principal token and certificate -service.credentialsExpiryMinutes int default=43200 # 30 days diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index b96e2112a5c..3351efff2dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -153,7 +153,7 @@ public class ControllerTest { .region("deep-space-9") .build(); try { - tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, applicationPackage.zippedContent(), new byte[0]); + tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, 2, applicationPackage.zippedContent(), new byte[0]); fail("Expected exception due to illegal deployment spec."); } catch (IllegalArgumentException e) { @@ -320,7 +320,7 @@ public class ControllerTest { tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.applications().deactivate(app1.id(), ZoneId.from(Environment.test, RegionName.from("us-east-1"))); tester.applications().deactivate(app1.id(), ZoneId.from(Environment.staging, RegionName.from("us-east-3"))); - tester.applications().deleteApplication(app1.id(), Optional.of(new NToken("ntoken"))); + tester.applications().deleteApplication(app1.id(), Optional.of(new OktaAccessToken("okta-token"))); try (RotationLock lock = tester.applications().rotationRepository().lock()) { assertTrue("Rotation is unassigned", tester.applications().rotationRepository().availableRotations(lock) @@ -445,6 +445,29 @@ public class ControllerTest { tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty()); } + @Test + public void testSuspension() { + DeploymentTester tester = new DeploymentTester(); + Application app = tester.createApplication("app1", "tenant1", 1, 11L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .region("us-east-3") + .build(); + SourceRevision source = new SourceRevision("repo", "master", "commit1"); + + ApplicationVersion applicationVersion = ApplicationVersion.from(source, 101); + runDeployment(tester, app.id(), applicationVersion, applicationPackage, source,101); + + DeploymentId deployment1 = new DeploymentId(app.id(), ZoneId.from(Environment.prod, RegionName.from("corp-us-east-1"))); + DeploymentId deployment2 = new DeploymentId(app.id(), ZoneId.from(Environment.prod, RegionName.from("us-east-3"))); + assertFalse(tester.configServer().isSuspended(deployment1)); + assertFalse(tester.configServer().isSuspended(deployment2)); + tester.configServer().setSuspended(deployment1, true); + assertTrue(tester.configServer().isSuspended(deployment1)); + assertFalse(tester.configServer().isSuspended(deployment2)); + } + private void runUpgrade(DeploymentTester tester, ApplicationId application, ApplicationVersion version) { Version next = Version.fromString("6.2"); tester.upgradeSystem(next); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 287fbe8c36d..8994c68acf3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; @@ -247,14 +248,14 @@ public final class ControllerTester { Optional.ofNullable(propertyId) .map(Object::toString) .map(PropertyId::new)); - controller().tenants().create(tenant, TestIdentities.userNToken); + controller().tenants().create(tenant, new OktaAccessToken("okta-token")); assertNotNull(controller().tenants().tenant(name)); return name; } public Application createApplication(TenantName tenant, String applicationName, String instanceName, long projectId) { ApplicationId applicationId = ApplicationId.from(tenant.value(), applicationName, instanceName); - controller().applications().createApplication(applicationId, Optional.of(TestIdentities.userNToken)); + controller().applications().createApplication(applicationId, Optional.of(new OktaAccessToken("okta-token"))); controller().applications().lockOrThrow(applicationId, lockedApplication -> controller().applications().store(lockedApplication.withProjectId(OptionalLong.of(projectId)))); return controller().applications().require(applicationId); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java deleted file mode 100644 index 18d3e92620d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.vespa.athenz.api.NToken; -import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; -import com.yahoo.vespa.hosted.controller.api.identifiers.Property; -import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; - -/** - * @author Tony Vaagenes - */ -public class TestIdentities { - - public static final EnvironmentId environment = new EnvironmentId("dev"); - - public static final RegionId region = new RegionId("us-east-1"); - - public static final InstanceId instance = new InstanceId("default"); - - public static final Property property = new Property("property"); - - public static final NToken userNToken = new NToken("dummy"); - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 0c0953371c5..70a47934262 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; @@ -10,8 +11,10 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -30,7 +33,9 @@ import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Optional; import java.util.function.Supplier; @@ -112,6 +117,68 @@ public class DeploymentTriggerTest { } @Test + public void abortsInternalJobsOnNewApplicationChange() { + InternalDeploymentTester iTester = new InternalDeploymentTester(); + tester = iTester.tester(); + + Application app = iTester.app(); + ApplicationPackage applicationPackage = InternalDeploymentTester.applicationPackage; + + tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit(); + tester.deployAndNotify(app, true, systemTest); + tester.deployAndNotify(app, true, stagingTest); + tester.assertRunning(productionUsCentral1, app.id()); + + // Jobs run externally are not affected. + tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit(); + tester.assertRunning(productionUsCentral1, app.id()); + + tester.applications().deploymentTrigger().cancelChange(app.id(), false); + tester.deployAndNotify(app, false, systemTest); + tester.deployAndNotify(app, false, stagingTest); + tester.deployAndNotify(app, false, productionUsCentral1); + assertEquals(Change.empty(), tester.application(app.id()).change()); + tester.assertNotRunning(systemTest, app.id()); + tester.assertNotRunning(stagingTest, app.id()); + tester.assertNotRunning(productionUsCentral1, app.id()); + + RunId id = iTester.newRun(productionUsCentral1); + assertTrue(iTester.jobs().active(id).isPresent()); + + // Jobs run internally are aborted. + iTester.newSubmission(); + assertTrue(iTester.jobs().active(id).isPresent()); + iTester.runner().run(); + assertFalse(iTester.jobs().active(id).isPresent()); + + tester.readyJobTrigger().maintain(); + assertEquals(EnumSet.of(systemTest, stagingTest), iTester.jobs().active().stream() + .map(run -> run.id().type()) + .collect(Collectors.toCollection(() -> EnumSet.noneOf(JobType.class)))); + + iTester.runJob(JobType.systemTest); + iTester.runJob(JobType.stagingTest); + tester.readyJobTrigger().maintain(); + iTester.runJob(JobType.productionUsCentral1); + tester.readyJobTrigger().maintain(); + iTester.runJob(JobType.productionUsWest1); + iTester.runJob(JobType.productionUsEast3); + assertEquals(Change.empty(), iTester.app().change()); + + tester.upgradeSystem(new Version("8.9")); + iTester.runJob(JobType.systemTest); + iTester.runJob(JobType.stagingTest); + tester.readyJobTrigger().maintain(); + + // Jobs run internally are not aborted when the new submission is delayed. + iTester.newSubmission(); + iTester.runner().run(); + assertEquals(EnumSet.of(productionUsCentral1), iTester.jobs().active().stream() + .map(run -> run.id().type()) + .collect(Collectors.toCollection(() -> EnumSet.noneOf(JobType.class)))); + } + + @Test public void deploymentSpecDecidesTriggerOrder() { TenantName tenant = tester.controllerTester().createTenant("tenant1", "domain1", 1L); MockBuildService mockBuildService = tester.buildService(); @@ -238,6 +305,49 @@ public class DeploymentTriggerTest { } @Test + public void testNoOtherChangesDuringSuspension() { + // Application is deployed in 3 regions: + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-central-1") + .parallel("us-west-1", "us-east-3") + .build(); + tester.jobCompletion(component) + .application(application) + .uploadArtifact(applicationPackage) + .submit(); + tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsWest1); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsEast3); + + // The first production zone is suspended: + tester.configServer().setSuspended(new DeploymentId(application.id(), JobType.productionUsCentral1.zone(tester.controller().system())), true); + + // A new change needs to be pushed out, but should not go beyond the suspended zone: + tester.jobCompletion(component) + .application(application) + .nextBuildNumber() + .sourceRevision(new SourceRevision("repository1", "master", "cafed00d")) + .uploadArtifact(applicationPackage) + .submit(); + tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1); + tester.triggerUntilQuiescence(); + tester.assertNotRunning(JobType.productionUsEast3, application.id()); + tester.assertNotRunning(JobType.productionUsWest1, application.id()); + + // The zone is unsuspended so jobs start: + tester.configServer().setSuspended(new DeploymentId(application.id(), JobType.productionUsCentral1.zone(tester.controller().system())), false); + tester.triggerUntilQuiescence(); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsWest1); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsEast3); + } + + @Test public void parallelDeploymentCompletesOutOfOrder() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) @@ -754,6 +864,7 @@ public class DeploymentTriggerTest { assertTrue("All jobs consumed", tester.buildService().jobs().isEmpty()); assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures()); } + @Test public void testRetriesJobsFailingForCurrentChange() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -843,25 +954,6 @@ public class DeploymentTriggerTest { } @Test - public void ignoresPullRequestInstances() throws Exception { - tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); - - // Current system version, matches version in test data - Version version = Version.fromString("6.42.1"); - tester.upgradeSystem(version); - assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); - - // Load test data data - byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json")); - Slime slime = SlimeUtils.jsonToSlime(json); - tester.controllerTester().createApplication(slime); - - // Failure redeployer does not restart deployment - tester.readyJobTrigger().maintain(); - assertTrue("No jobs scheduled", tester.buildService().jobs().isEmpty()); - } - - @Test public void applicationWithoutProjectIdIsNotTriggered() throws Exception { // Current system version, matches version in test data Version version = Version.fromString("6.42.1"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index d5bcb205641..0549a2e4226 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -82,7 +82,7 @@ public class InternalDeploymentTester { * Submits a new application, and returns the version of the new submission. */ public ApplicationVersion newSubmission() { - ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, applicationPackage.zippedContent(), new byte[0]); + ApplicationVersion version = jobs.submit(appId, BuildJob.defaultSourceRevision, 2, applicationPackage.zippedContent(), new byte[0]); tester.applicationStore().putApplicationPackage(appId, version.id(), applicationPackage.zippedContent()); tester.applicationStore().putTesterPackage(testerOf(appId), version.id(), new byte[0]); return version; @@ -109,7 +109,7 @@ public class InternalDeploymentTester { ApplicationVersion applicationVersion = newSubmission(); assertFalse(app().deployments().values().stream() - .anyMatch(deployment -> deployment.applicationVersion().equals(applicationVersion))); + .anyMatch(deployment -> deployment.applicationVersion().equals(applicationVersion))); assertEquals(applicationVersion, app().change().application().get()); assertFalse(app().change().platform().isPresent()); @@ -128,7 +128,7 @@ public class InternalDeploymentTester { public void deployNewPlatform(Version version) { tester.upgradeSystem(version); assertFalse(app().deployments().values().stream() - .anyMatch(deployment -> deployment.version().equals(version))); + .anyMatch(deployment -> deployment.version().equals(version))); assertEquals(version, app().change().platform().get()); assertFalse(app().change().application().isPresent()); @@ -230,7 +230,7 @@ public class InternalDeploymentTester { * Creates and submits a new application, and then starts the job of the given type. */ public RunId newRun(JobType type) { - assertFalse(app().deploymentJobs().builtInternally()); // Use this only once per test. + assertFalse(app().deploymentJobs().deployedInternally()); // Use this only once per test. newSubmission(); tester.readyJobTrigger().maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/AthenzFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/AthenzFilterMock.java index 4acd3a34c8d..72027234b28 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/AthenzFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/AthenzFilterMock.java @@ -13,7 +13,6 @@ import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.yolean.chain.Before; @@ -24,7 +23,6 @@ import com.yahoo.yolean.chain.Before; public class AthenzFilterMock implements SecurityRequestFilter { public static final String IDENTITY_HEADER_NAME = "Athenz-Identity"; - public static final String ATHENZ_NTOKEN_HEADER_NAME = "Athenz-NToken"; private static final ObjectMapper mapper = new ObjectMapper(); @@ -32,7 +30,6 @@ public class AthenzFilterMock implements SecurityRequestFilter { public void filter(DiscFilterRequest request, ResponseHandler handler) { if (request.getMethod().equalsIgnoreCase("OPTIONS")) return; String identityName = request.getHeader(IDENTITY_HEADER_NAME); - String nToken = request.getHeader(ATHENZ_NTOKEN_HEADER_NAME); if (identityName == null) { Response response = new Response(HttpResponse.Status.UNAUTHORIZED); response.headers().put("Content-Type", "application/json"); @@ -45,10 +42,7 @@ public class AthenzFilterMock implements SecurityRequestFilter { } } else { AthenzIdentity identity = AthenzIdentities.from(identityName); - AthenzPrincipal principal = - nToken == null ? - new AthenzPrincipal(identity) : - new AthenzPrincipal(identity, new NToken(nToken)); + AthenzPrincipal principal = new AthenzPrincipal(identity); request.setUserPrincipal(principal); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index e29ffd824cd..f011baef39a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -53,6 +54,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final NodeRepositoryMock nodeRepository = new NodeRepositoryMock(); private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>(); private final Version initialVersion = new Version(6, 1, 0); + private final Set<DeploymentId> suspendedApplications = new HashSet<>(); private Version lastPrepareVersion = null; private RuntimeException prepareException = null; @@ -153,6 +155,13 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return Optional.ofNullable(applications.get(id)); } + public void setSuspended(DeploymentId deployment, boolean suspend) { + if (suspend) + suspendedApplications.add(deployment); + else + suspendedApplications.remove(deployment); + } + @Override public NodeRepositoryMock nodeRepository() { return nodeRepository; @@ -236,6 +245,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override + public boolean isSuspended(DeploymentId deployment) { + return suspendedApplications.contains(deployment); + } + + @Override public void restart(DeploymentId deployment, Optional<Hostname> hostname) { nodeRepository().requestRestart(deployment, hostname.map(Identifier::id).map(HostName::from)); } @@ -252,17 +266,21 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) { + String cfgHostname = String.format("https://cfg.%s.%s.test.vip:4443", environment, region); + String cfgServiceUrlPrefix = String.format("%s/serviceview/v1/tenant/%s/application/%s/environment/%s/region/%s/instance/%s/service", + cfgHostname, tenantName, applicationName, + environment, region, instanceName); ApplicationView applicationView = new ApplicationView(); ClusterView cluster = new ClusterView(); cluster.name = "cluster1"; cluster.type = "content"; - cluster.url = "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1"; + cluster.url = cfgServiceUrlPrefix + "/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1"; ServiceView service = new ServiceView(); service.configId = "cluster1/storage/0"; service.host = "host1"; service.serviceName = "storagenode"; service.serviceType = "storagenode"; - service.url = "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/"; + service.url = cfgServiceUrlPrefix + "/storagenode-awe3slno6mmq2fye191y324jl/state/v1/"; cluster.services = new ArrayList<>(); cluster.services.add(service); applicationView.clusters = new ArrayList<>(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index cb6d971c2d9..ad6c26322da 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -117,12 +117,21 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public List<URI> getConfigServerUris(ZoneId zoneId) { - return Collections.singletonList(URI.create(String.format("https://cfg.%s.test:4443", zoneId.value()))); + return Collections.singletonList(URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value()))); } @Override public Optional<URI> getConfigServerVipUri(ZoneId zoneId) { - return Optional.of(URI.create(String.format("https://cfg.%s.test:4443", zoneId.value()))); + return Optional.of(URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value()))); + } + + @Override + public List<URI> getConfigServerApiUris(ZoneId zoneId) { + List<URI> uris = new ArrayList<URI>(); + uris.add(URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value()))); + uris.add(URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value()))); + + return uris; } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java index 66be2d09797..71d2690bb4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainerTest.java @@ -3,24 +3,45 @@ package com.yahoo.vespa.hosted.controller.maintenance; // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.stubbing.ServeEvent; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; import java.time.Duration; +import java.util.List; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.findAll; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents; +import static com.github.tomakehurst.wiremock.client.WireMock.okJson; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static org.junit.Assert.assertEquals; /** * @author smorgrav */ public class ClusterUtilizationMaintainerTest { + @Rule + public WireMockRule wireMockRule = new WireMockRule(4443); + @Test public void maintain() { + wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/clusterutilization")) + .willReturn(aResponse().withStatus(200))); ControllerTester tester = new ControllerTester(); ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); @@ -28,12 +49,16 @@ public class ClusterUtilizationMaintainerTest { Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); Assert.assertEquals(0, deployment.clusterUtils().size()); - ClusterUtilizationMaintainer mainainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb())); - mainainer.maintain(); + ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/"); + ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder); + ClusterUtilizationMaintainer maintainer = new ClusterUtilizationMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()), apiAuthorityConfig); + maintainer.maintain(); - deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); - Assert.assertEquals(1, deployment.clusterUtils().size()); - Assert.assertEquals(0.5554, deployment.clusterUtils().get(ClusterSpec.Id.from("default")).getCpu(), Double.MIN_VALUE); + List<ServeEvent> allServeEvents = getAllServeEvents(); + assertEquals(allServeEvents.size(), 1); + LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/clusterutilization"))).get(0); + String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"deployments\":[{\"zoneId\":\"dev.us-east-1\",\"clusterUtil\":[{\"clusterSpecId\":\"default\",\"cpu\":0.5554,\"memory\":0.6990000000000001,\"disk\":0.34590000000000004,\"diskBusy\":0.0}]}]}]"; + assertEquals(expectedBody, new String(request.getBody())); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index e11440a372c..b0d28e05744 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -1,27 +1,30 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.ApplicationId; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.stubbing.ServeEvent; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.RotationStatus; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock; +import org.junit.Rule; import org.junit.Test; import java.time.Duration; -import java.time.Instant; -import java.util.function.Supplier; - -import static java.time.temporal.ChronoUnit.MILLIS; +import java.util.List; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.findAll; +import static com.github.tomakehurst.wiremock.client.WireMock.getAllServeEvents; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; /** * @author smorgrav @@ -29,69 +32,16 @@ import static org.junit.Assert.assertFalse; */ public class DeploymentMetricsMaintainerTest { - @Test - public void updates_metrics() { - ControllerTester tester = new ControllerTester(); - ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1", - Environment.dev, 123).id(); - DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); - Supplier<Application> app = tester.application(appId); - Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get(); - - // No metrics gathered yet - assertEquals(0, app.get().metrics().queryServiceQuality(), 0); - assertEquals(0, deployment.get().metrics().documentCount(), 0); - assertFalse("Never received any queries", deployment.get().activity().lastQueried().isPresent()); - assertFalse("Never received any writes", deployment.get().activity().lastWritten().isPresent()); - - // Metrics are gathered and saved to application - maintainer.maintain(); - assertEquals(0.5, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE); - assertEquals(0.7, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE); - assertEquals(1, deployment.get().metrics().queriesPerSecond(), Double.MIN_VALUE); - assertEquals(2, deployment.get().metrics().writesPerSecond(), Double.MIN_VALUE); - assertEquals(3, deployment.get().metrics().documentCount(), Double.MIN_VALUE); - assertEquals(4, deployment.get().metrics().queryLatencyMillis(), Double.MIN_VALUE); - assertEquals(5, deployment.get().metrics().writeLatencyMillis(), Double.MIN_VALUE); - Instant t1 = tester.clock().instant().truncatedTo(MILLIS); - assertEquals(t1, deployment.get().activity().lastQueried().get()); - assertEquals(t1, deployment.get().activity().lastWritten().get()); - - // Time passes. Activity is updated as app is still receiving traffic - tester.clock().advance(Duration.ofHours(1)); - Instant t2 = tester.clock().instant().truncatedTo(MILLIS); - maintainer.maintain(); - assertEquals(t2, deployment.get().activity().lastQueried().get()); - assertEquals(t2, deployment.get().activity().lastWritten().get()); - assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE); - assertEquals(2, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE); - - // Query traffic disappears. Query activity stops updating - tester.clock().advance(Duration.ofHours(1)); - Instant t3 = tester.clock().instant().truncatedTo(MILLIS); - tester.metricsService().setMetric("queriesPerSecond", 0D); - tester.metricsService().setMetric("writesPerSecond", 5D); - maintainer.maintain(); - assertEquals(t2, deployment.get().activity().lastQueried().get()); - assertEquals(t3, deployment.get().activity().lastWritten().get()); - assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE); - assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE); - - // Feed traffic disappears. Feed activity stops updating - tester.clock().advance(Duration.ofHours(1)); - tester.metricsService().setMetric("writesPerSecond", 0D); - maintainer.maintain(); - assertEquals(t2, deployment.get().activity().lastQueried().get()); - assertEquals(t3, deployment.get().activity().lastWritten().get()); - assertEquals(1, deployment.get().activity().lastQueriesPerSecond().getAsDouble(), Double.MIN_VALUE); - assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE); - } + @Rule + public WireMockRule wireMockRule = new WireMockRule(4443); @Test - public void updates_rotation_status() { + public void maintain() { DeploymentTester tester = new DeploymentTester(); MetricsServiceMock metricsService = tester.controllerTester().metricsService(); - DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); + ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/"); + ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder); + DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), apiAuthorityConfig); Application application = tester.createApplication("app1", "tenant1", 1, 1L); ZoneId zone1 = ZoneId.from("prod", "us-west-1"); ZoneId zone2 = ZoneId.from("prod", "us-east-3"); @@ -105,32 +55,22 @@ public class DeploymentMetricsMaintainerTest { .build(); tester.deployCompletely(application, applicationPackage); - Supplier<Application> app = () -> tester.application(application.id()); - Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1); - Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2); String assignedRotation = "rotation-fqdn-01"; tester.controllerTester().metricsService().addRotation(assignedRotation); - // No status gathered yet - assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment1.get())); - assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment2.get())); - // One rotation out, one in metricsService.setZoneIn(assignedRotation, "proxy.prod.us-west-1.vip.test"); metricsService.setZoneOut(assignedRotation,"proxy.prod.us-east-3.vip.test"); - maintainer.maintain(); - assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get())); - assertEquals(RotationStatus.out, app.get().rotationStatus(deployment2.get())); - // All rotations in - metricsService.setZoneIn(assignedRotation,"proxy.prod.us-east-3.vip.test"); + wireMockRule.stubFor(post(urlEqualTo("/metricforwarding/v1/deploymentmetrics/")) + .willReturn(aResponse().withStatus(200))); maintainer.maintain(); - assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get())); - assertEquals(RotationStatus.in, app.get().rotationStatus(deployment2.get())); - } - private static DeploymentMetricsMaintainer maintainer(Controller controller) { - return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator())); + List<ServeEvent> allServeEvents = getAllServeEvents(); + assertEquals(1, allServeEvents.size()); + LoggedRequest request = findAll(postRequestedFor(urlEqualTo("/metricforwarding/v1/deploymentmetrics/"))).get(0); + String expectedBody = "[{\"applicationId\":\"tenant1:app1:default\",\"applicationMetrics\":{\"queryServiceQuality\":0.5,\"writeServiceQuality\":0.7},\"rotationStatus\":[{\"hostname\":\"proxy.prod.us-east-3.vip.test\",\"rotationStatus\":\"out\"},{\"hostname\":\"proxy.prod.us-west-1.vip.test\",\"rotationStatus\":\"in\"}],\"deploymentMetrics\":[{\"zoneId\":\"prod.us-west-1\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0},{\"zoneId\":\"prod.us-east-3\",\"queriesPerSecond\":1.0,\"writesPerSecond\":2.0,\"documentCount\":3.0,\"queryLatencyMillis\":4.0,\"writeLatencyMillis\":5.0}]}]"; + assertEquals(expectedBody, new String(request.getBody())); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index b950e969300..92f3e87f001 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; @@ -88,7 +88,7 @@ public class DnsMaintainerTest { tester.deployAndNotify(application, applicationPackage, true, systemTest); tester.applications().deactivate(application.id(), ZoneId.from(Environment.test, RegionName.from("us-east-1"))); tester.applications().deactivate(application.id(), ZoneId.from(Environment.staging, RegionName.from("us-east-3"))); - tester.applications().deleteApplication(application.id(), Optional.of(new NToken("ntoken"))); + tester.applications().deleteApplication(application.id(), Optional.of(new OktaAccessToken("okta-token"))); // DnsMaintainer removes records dnsMaintainer.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index e506c8c56ca..ca31eb52979 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.TestIdentities; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.SourceRevision; @@ -49,11 +49,11 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; -import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -82,7 +82,7 @@ public class JobRunnerTest { phasedExecutor(phaser), stepRunner); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); jobs.start(id, systemTest, versions); try { @@ -113,7 +113,7 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); jobs.start(id, systemTest, versions); @@ -197,7 +197,7 @@ public class JobRunnerTest { Executors.newFixedThreadPool(32), waitingRunner(barrier)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); RunId runId = new RunId(id, systemTest, 1); jobs.start(id, systemTest, versions); @@ -211,7 +211,7 @@ public class JobRunnerTest { // Thread is still trying to deploy tester -- delete application, and see all data is garbage collected. assertEquals(Collections.singletonList(runId), jobs.active().stream().map(run -> run.id()).collect(Collectors.toList())); - tester.controller().applications().deleteApplication(id, Optional.of(TestIdentities.userNToken)); + tester.controller().applications().deleteApplication(id, Optional.of(new OktaAccessToken("okta-token"))); assertEquals(Collections.emptyList(), jobs.active()); assertEquals(runId, jobs.last(id, systemTest).get().id()); @@ -233,7 +233,7 @@ public class JobRunnerTest { inThreadExecutor(), (id, step) -> Optional.of(running)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); for (int i = 0; i < jobs.historyLength(); i++) { jobs.start(id, systemTest, versions); @@ -261,7 +261,7 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), 2, new byte[0], new byte[0]); jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json deleted file mode 100644 index 425b9d4512d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "id": "tenant1:app1:default-pr1", - "deploymentSpecField": "<deployment version='1.0'>\n <test />\n <staging />\n <prod global-service-id=\"default\">\n <region active=\"true\">us-central-1</region>\n </prod>\n</deployment>\n", - "validationOverrides": "<deployment version='1.0'/>", - "deployments": [], - "deploymentJobs": { - "projectId": 42, - "jobStatus": [ - { - "jobType": "system-test", - "lastTriggered": { - "version": "6.42.1", - "revision": { - "applicationPackageHash": "dead" - }, - "upgrade": false, - "at": 1499075576005 - } - } - ] - }, - "outstandingChangeField": false -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index b571e6f1c48..11e8a82dd42 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -6,16 +6,16 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.TestIdentities; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -77,9 +77,9 @@ public class ContainerControllerTester { controller().tenants().create(AthenzTenant.create(TenantName.from(tenant), domain1, new Property("property1"), Optional.of(new PropertyId("1234"))), - TestIdentities.userNToken); + new OktaAccessToken("okta-token")); ApplicationId app = ApplicationId.from(tenant, application, "default"); - return controller().applications().createApplication(app, Optional.of(TestIdentities.userNToken)); + return controller().applications().createApplication(app, Optional.of(new OktaAccessToken("okta-token"))); } public Application deploy(Application application, ApplicationPackage applicationPackage, ZoneId zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index e47cdf887f9..6044dd7b8f8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -7,7 +7,7 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import org.junit.After; import org.junit.Before; @@ -15,7 +15,6 @@ import org.junit.Before; import java.io.UncheckedIOException; import java.nio.charset.CharacterCodingException; -import static com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock.ATHENZ_NTOKEN_HEADER_NAME; import static com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock.IDENTITY_HEADER_NAME; import static org.junit.Assert.assertEquals; @@ -96,6 +95,9 @@ public class ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.contactinfo.ContactInfoHandler'>\n" + " <binding>http://*/contactinfo/v1/*</binding>\n" + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.metrics.MetricForwardingApiHandler'>\n" + + " <binding>http://*/metricforwarding/v1/*</binding>\n" + + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>\n" + " <binding>http://*/deployment/v1/*</binding>\n" + " </handler>\n" + @@ -152,8 +154,8 @@ public class ControllerContainerTest { return request; } - protected static Request addNTokenToRequest(Request request, NToken nToken) { - request.getHeaders().put(ATHENZ_NTOKEN_HEADER_NAME, nToken.getRawToken()); + protected static Request addOktaAccessToken(Request request, OktaAccessToken token) { + request.getHeaders().put(OktaAccessToken.HTTP_HEADER_NAME, token.token()); return request; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index d620d78d3f0..f840c188a34 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Cursor; @@ -16,17 +17,18 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -41,8 +43,10 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; @@ -75,6 +79,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeMap; import java.util.function.Supplier; import static com.yahoo.application.container.handler.Request.Method.DELETE; @@ -110,7 +115,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private static final ScrewdriverId SCREWDRIVER_ID = new ScrewdriverId("12345"); private static final UserId USER_ID = new UserId("myuser"); private static final UserId HOSTED_VESPA_OPERATOR = new UserId("johnoperator"); - private static final NToken N_TOKEN = new NToken("dummy"); + private static final OktaAccessToken OKTA_AT = new OktaAccessToken("dummy"); private static final ZoneId TEST_ZONE = ZoneId.from(Environment.test, RegionName.from("us-east-1")); private static final ZoneId STAGING_ZONE = ZoneId.from(Environment.staging, RegionName.from("us-east-3")); @@ -146,12 +151,12 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1", POST) .userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json")); // PUT (modify) a tenant tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) .userIdentity(USER_ID) - .nToken(N_TOKEN) + .oktaAccessToken(OKTA_AT) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // GET the authenticated user (with associated tenants) @@ -170,13 +175,13 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (add) a tenant with property ID tester.assertResponse(request("/application/v4/tenant/tenant2", POST) .userIdentity(USER_ID) - .nToken(N_TOKEN) + .oktaAccessToken(OKTA_AT) .data("{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}"), new File("tenant-without-applications-with-id.json")); // PUT (modify) a tenant with property ID tester.assertResponse(request("/application/v4/tenant/tenant2", PUT) .userIdentity(USER_ID) - .nToken(N_TOKEN) + .oktaAccessToken(OKTA_AT) .data("{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}"), new File("tenant-without-applications-with-id.json")); // GET a tenant with property ID and contact information @@ -187,7 +192,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (create) an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("application-reference.json")); // GET a tenant tester.assertResponse(request("/application/v4/tenant/tenant1", GET).userIdentity(USER_ID), @@ -267,7 +272,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", POST) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("application-reference-2.json")); ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); @@ -293,7 +298,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE application tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", DELETE) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), ""); // GET tenant screwdriver projects @@ -361,6 +366,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .screwdriverIdentity(SCREWDRIVER_ID), "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"No node with the hostname host1 is known.\"}", 500); + // GET suspended + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/suspended", GET) + .userIdentity(USER_ID), + new File("suspended.json")); + // GET services tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", GET) .userIdentity(USER_ID), @@ -419,6 +429,10 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(createApplicationSubmissionData(packageWithService)), "{\"version\":\"1.0.43-d00d\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Nothing to abort.\"}"); + // PUT (create) the authenticated user byte[] data = new byte[0]; tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) @@ -445,11 +459,11 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE).userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), ""); // DELETE a tenant tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE).userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json")); } @@ -521,13 +535,13 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create tenant tester.assertResponse(request("/application/v4/tenant/tenant1", POST).userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json")); // Create application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("application-reference.json")); // Grant deploy access @@ -662,21 +676,21 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1", POST) .userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json")); // POST (add) another tenant under the same domain tester.assertResponse(request("/application/v4/tenant/tenant2", POST) .userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create tenant 'tenant2': The Athens domain 'domain1' is already connected to tenant 'tenant1'\"}", 400); // Add the same tenant again tester.assertResponse(request("/application/v4/tenant/tenant1", POST) .userIdentity(USER_ID) - .nToken(N_TOKEN) + .oktaAccessToken(OKTA_AT) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'tenant1' already exists\"}", 400); @@ -685,7 +699,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/my_tenant_2", POST) .userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"New tenant or application names must start with a letter, may contain no more than 20 characters, and may only contain lowercase letters, digits or dashes, but no double-dashes.\"}", 400); @@ -693,14 +707,14 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/by-tenant2", POST) .userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Athenz tenant name cannot have prefix 'by-'\"}", 400); // POST (create) an (empty) application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("application-reference.json")); // Create the same application again @@ -743,14 +757,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE tenant which has an application tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not delete tenant 'tenant1': This tenant has active applications\"}", 400); // DELETE application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), ""); // DELETE application again - should produce 404 tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) @@ -760,7 +774,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE tenant tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json")); // DELETE tenant again - should produce 404 tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) @@ -777,12 +791,12 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create legancy tenant name containing underscores tester.controller().tenants().create(new AthenzTenant(TenantName.from("my_tenant"), ATHENZ_TENANT_DOMAIN, new Property("property1"), Optional.empty(), Optional.empty()), - N_TOKEN); + OKTA_AT); // POST (add) a Athenz tenant with dashes duplicates existing one with underscores tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'my-tenant' already exists\"}", 400); } @@ -818,21 +832,21 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1", POST) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") .userIdentity(authorizedUser) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json"), 200); // Creating an application for an Athens domain the user is not admin for is disallowed tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) .userIdentity(unauthorizedUser) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), "{\n \"code\" : 403,\n \"message\" : \"Tenant admin or Vespa operator role required\"\n}", 403); // (Create it with the right tenant id) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) .userIdentity(authorizedUser) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("application-reference.json"), 200); @@ -853,7 +867,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // (Deleting it with the right tenant id) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) .userIdentity(authorizedUser) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), "", 200); @@ -869,7 +883,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) .data("{\"athensDomain\":\"domain2\", \"property\":\"property1\"}") .userIdentity(authorizedUser) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), "{\"tenant\":\"tenant1\",\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[]}", 200); @@ -1084,7 +1098,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private final Request.Method method; private byte[] data = new byte[0]; private AthenzIdentity identity; - private NToken nToken; + private OktaAccessToken oktaAccessToken; private String contentType = "application/json"; private String recursive; @@ -1106,7 +1120,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } private RequestBuilder userIdentity(UserId userId) { this.identity = HostedAthenzIdentities.from(userId); return this; } private RequestBuilder screwdriverIdentity(ScrewdriverId screwdriverId) { this.identity = HostedAthenzIdentities.from(screwdriverId); return this; } - private RequestBuilder nToken(NToken nToken) { this.nToken = nToken; return this; } + private RequestBuilder oktaAccessToken(OktaAccessToken oktaAccessToken) { this.oktaAccessToken = oktaAccessToken; return this; } private RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; } private RequestBuilder recursive(String recursive) { this.recursive = recursive; return this; } @@ -1120,8 +1134,8 @@ public class ApplicationApiTest extends ControllerContainerTest { if (identity != null) { addIdentityToRequest(request, identity); } - if (nToken != null) { - addNTokenToRequest(request, nToken); + if (oktaAccessToken != null) { + addOktaAccessToken(request, oktaAccessToken); } return request; } @@ -1164,11 +1178,11 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1", POST) .userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) .userIdentity(USER_ID) - .nToken(N_TOKEN), + .oktaAccessToken(OKTA_AT), new File("application-reference.json")); addScrewdriverUserToDeployRole(SCREWDRIVER_ID, ATHENZ_TENANT_DOMAIN, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId("application1")); @@ -1264,8 +1278,22 @@ public class ApplicationApiTest extends ControllerContainerTest { String vipName = "proxy." + zone.value() + ".vip.test"; metricsService().addRotation(rotationName) .setZoneIn(rotationName, vipName); - - new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())).run(); + ApplicationController applicationController = controllerTester.controller().applications(); + List<Application> applicationList = applicationController.asList(); + applicationList.stream().forEach(application -> { + applicationController.lockIfPresent(application.id(), locked -> + applicationController.store(locked.withRotationStatus(rotationStatus(application)))); + });} + + private Map<HostName, RotationStatus> rotationStatus(Application application) { + return controllerTester.controller().applications().rotationRepository().getRotation(application) + .map(rotation -> controllerTester.controller().metricsService().getRotationStatus(rotation.name())) + .map(rotationStatus -> { + Map<HostName, RotationStatus> result = new TreeMap<>(); + rotationStatus.forEach((hostname, status) -> result.put(hostname, RotationStatus.in)); + return result; + }) + .orElseGet(Collections::emptyMap); } private void updateContactInformation() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 7d379911238..293a1f8a746 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -48,6 +48,7 @@ public class JobControllerApiHandlerHelperTest { // Revision 1 gets deployed everywhere. ApplicationVersion revision1 = tester.deployNewSubmission(); + assertEquals(2, tester.app().deploymentJobs().projectId().getAsLong()); tester.clock().advance(Duration.ofMillis(1000)); // Revision 2 gets deployed everywhere except in us-east-3. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index 07a3dbb7f95..7bb78aba459 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -2,6 +2,7 @@ "application": "application1", "instance": "default", "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "deployedInternally": false, "deploymentJobs": [ { "type": "component", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 0d7607f1df6..2f25e9471aa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -12,6 +12,7 @@ } } }, + "deployedInternally": false, "deploymentJobs": [ { "type": "component", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index 4e4a870662a..70da148ef86 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -12,6 +12,7 @@ } } }, + "deployedInternally": false, "deploymentJobs": [ { "type": "component", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json index 837c46aaec1..56abc8f9d82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json @@ -15,6 +15,7 @@ } } }, + "deployedInternally": false, "deploymentJobs": [ { "type": "component", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json index 8a0849393c9..c68b7dc9e87 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json @@ -3,10 +3,10 @@ { "name": "cluster1", "type": "content", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", "services": [ { - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", "serviceType": "storagenode", "serviceName": "storagenode", "configId": "cluster1/storage/0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/suspended.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/suspended.json new file mode 100644 index 00000000000..a360474da49 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/suspended.json @@ -0,0 +1,3 @@ +{ + "suspended":false +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index 9951e4499f7..c3b9c11de88 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -14,8 +14,8 @@ import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.HostedAthenzIdentities; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/metrics/MetricForwardingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/metrics/MetricForwardingApiHandlerTest.java new file mode 100644 index 00000000000..551644377fb --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/metrics/MetricForwardingApiHandlerTest.java @@ -0,0 +1,157 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.metrics; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.junit.Before; +import org.junit.Test; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author olaa + */ +public class MetricForwardingApiHandlerTest extends ControllerContainerTest { + + private static final double DELTA = 0.00001; + private ContainerControllerTester tester; + + @Before + public void before() { + tester = new ContainerControllerTester(container, null); + } + + @Test + public void testUpdatingDeploymentMetrics() { + ZoneId zoneId = ZoneId.from(Environment.test, RegionName.from("us-east-1")); + deployApplication(zoneId); + Application application = getUpdatedApplication(); + Deployment deployment = application.deployments().get(zoneId); + + // Verify deployment and system metrics are initially 0 + assertEquals(0, application.metrics().queryServiceQuality(), DELTA); + assertEquals(0, application.metrics().writeServiceQuality(), DELTA); + assertEquals(0, deployment.metrics().documentCount(), DELTA); + assertEquals(0, deployment.metrics().queriesPerSecond(), DELTA); + assertEquals(0, deployment.metrics().queryLatencyMillis(), DELTA); + assertEquals(0, deployment.metrics().writeLatencyMillis(), DELTA); + assertEquals(0, deployment.metrics().writesPerSecond(), DELTA); + assertFalse(application.rotationStatus().containsKey(HostName.from("host1"))); + assertFalse(application.rotationStatus().containsKey(HostName.from("host2"))); + + String deploymentMetrics = "[{\"applicationId\":\"tenant1:application1:default\"," + + "\"applicationMetrics\":{" + + " \"queryServiceQuality\":0.5," + + " \"writeServiceQuality\":0.7}," + + "\"rotationStatus\":[" + + " {" + + " \"hostname\":\"proxy.prod.us-east-3.vip.test\"," + + " \"rotationStatus\":\"out\"" + + " }," + + " {" + + " \"hostname\":\"proxy.prod.us-east-1.vip.test\"," + + " \"rotationStatus\":\"in\"" + + " }]," + + "\"deploymentMetrics\":[" + + " {" + + " \"zoneId\":\"test.us-east-1\"," + + " \"queriesPerSecond\":1.0," + + " \"writesPerSecond\":2.0," + + " \"documentCount\":3.0," + + " \"queryLatencyMillis\":4.0," + + " \"writeLatencyMillis\":5.0" + + " }" + + "]}]"; + String expectedResponseMessage = "Added deployment metrics"; + assertResponse(new Request("http://localhost:8080/metricforwarding/v1/deploymentmetrics", deploymentMetrics, Request.Method.POST), 200, expectedResponseMessage); + + // Verify that deployment metrics are updated + application = getUpdatedApplication(); + assertEquals(0.5, application.metrics().queryServiceQuality(), DELTA); + assertEquals(0.7, application.metrics().writeServiceQuality(), DELTA); + + deployment = application.deployments().get(zoneId); + assertEquals(3.0, deployment.metrics().documentCount(), DELTA); + assertEquals(1.0, deployment.metrics().queriesPerSecond(), DELTA); + assertEquals(4.0, deployment.metrics().queryLatencyMillis(), DELTA); + assertEquals(5.0, deployment.metrics().writeLatencyMillis(), DELTA); + assertEquals(2, deployment.metrics().writesPerSecond(), DELTA); + assertEquals(RotationStatus.in, application.rotationStatus().get(HostName.from("proxy.prod.us-east-1.vip.test"))); + assertEquals(RotationStatus.out, application.rotationStatus().get(HostName.from("proxy.prod.us-east-3.vip.test"))); + + } + + @Test + public void testUpdatingSystemMetrics() { + ZoneId zoneId = ZoneId.from(Environment.test, RegionName.from("us-east-1")); + deployApplication(zoneId); + Application application = getUpdatedApplication(); + Deployment deployment = application.deployments().get(zoneId); + assertFalse(deployment.clusterUtils().containsKey(ClusterSpec.Id.from("cluster1"))); + assertFalse(deployment.clusterUtils().containsKey(ClusterSpec.Id.from("cluster2"))); + String systemMetrics = "[{\"applicationId\":\"tenant1:application1:default\"," + + "\"deployments\":[{\"zoneId\":\"test.us-east-1\",\"clusterUtil\":[" + + "{" + + " \"clusterSpecId\":\"default\"," + + " \"cpu\":0.5554," + + " \"memory\":0.6990000000000001," + + " \"disk\":0.34590000000000004," + + " \"diskBusy\":0.0" + + "}," + + "{" + + " \"clusterSpecId\":\"cluster2\"," + + " \"cpu\":0.6," + + " \"memory\":0.8," + + " \"disk\":0.5," + + " \"diskBusy\":0.1" + + "}" + + "]}]}]"; + String expectedResponseMessage = "Added cluster utilization metrics"; + assertResponse(new Request("http://localhost:8080/metricforwarding/v1/clusterutilization", systemMetrics, Request.Method.POST), 200, expectedResponseMessage); + deployment = getUpdatedApplication().deployments().get(zoneId); + + ClusterUtilization clusterUtilization = deployment.clusterUtils().get(ClusterSpec.Id.from("default")); + assertEquals(0.5554, clusterUtilization.getCpu(), DELTA); + assertEquals(0.699, clusterUtilization.getMemory(), DELTA); + assertEquals(0.3459, clusterUtilization.getDisk(), DELTA); + assertEquals(0.0, clusterUtilization.getDiskBusy(), DELTA); + clusterUtilization = deployment.clusterUtils().get(ClusterSpec.Id.from("cluster2")); + assertEquals(0.6, clusterUtilization.getCpu(), DELTA); + assertEquals(0.8, clusterUtilization.getMemory(), DELTA); + assertEquals(0.5, clusterUtilization.getDisk(), DELTA); + assertEquals(0.1, clusterUtilization.getDiskBusy(), DELTA); + } + + private Application getUpdatedApplication() { + return tester.controller().applications().asList().get(0); + } + + private void deployApplication(ZoneId zoneId) { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(zoneId.environment()) + .region(zoneId.region().value()) + .build(); + Application application = tester.createApplication(); + tester.jobCompletion(JobType.component) + .application(application) + .projectId(123L) + .uploadArtifact(applicationPackage) + .submit(); + tester.deploy(application, applicationPackage, zoneId); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index 6cd4464dce4..6e5e8a25e22 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -8,10 +8,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import org.junit.Test; -import java.io.File; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.OptionalLong; @@ -22,23 +20,9 @@ import java.util.OptionalLong; */ public class ScrewdriverApiTest extends ControllerContainerTest { - private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/"; - - @Test - public void testGetReleaseStatus() throws Exception { - ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); - tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/screwdriver/v1/release/vespa"), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Information about the current system version is not available at this time\"}", - 404); - - tester.controller().updateVersionStatus(VersionStatus.compute(tester.controller())); - tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/screwdriver/v1/release/vespa"), - new File("release-response.json"), 200); - } - @Test public void testTriggerJobForApplication() { - ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); + ContainerControllerTester tester = new ContainerControllerTester(container, null); tester.containerTester().computeVersionStatus(); Application app = tester.createApplication(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json deleted file mode 100644 index 9d96e08e695..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "version":"(ignore)", - "sha":"(ignore)", - "date":0 -} diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java index a0516a62bd9..517f44cb983 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java @@ -10,6 +10,7 @@ import com.yahoo.document.Field; import com.yahoo.document.serialization.DocumentUpdateWriter; import com.yahoo.document.update.FieldUpdate; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -18,6 +19,7 @@ import java.util.Map; * * @author vegardh */ +// TODO Vespa 7 Remove all deprecated methods public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOperationWrapper { private DocumentUpdate docU; @@ -41,10 +43,12 @@ public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOpera @Override public FieldUpdate getFieldUpdate(Field field) { - return getFieldUpdate(field.getName()); + return docU.getFieldUpdate(field); } @Override + @Deprecated + @SuppressWarnings( "deprecation" ) public FieldUpdate getFieldUpdate(int index) { return docU.getFieldUpdate(index); } @@ -60,10 +64,15 @@ public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOpera } @Override + @Deprecated + @SuppressWarnings( "deprecation" ) public List<FieldUpdate> getFieldUpdates() { return docU.getFieldUpdates(); } - + @Override + public Collection<FieldUpdate> fieldUpdates() { + return docU.fieldUpdates(); + } @Override public DocumentId getId() { return docU.getId(); diff --git a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java index 9a3a29e55b1..a89dbfcc782 100644 --- a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java @@ -37,8 +37,10 @@ public class ProcessingUpdateTestCase { @Test public void testProcessingUpdates() { DocumentType articleType = new DocumentType("article"); - articleType.addField(new Field("body", DataType.STRING, true)); - articleType.addField(new Field("title", DataType.STRING, true)); + Field bodyField = new Field("body", DataType.STRING, true); + Field titleField = new Field("title", DataType.STRING, true); + articleType.addField(bodyField); + articleType.addField(titleField); dtm = new DocumentTypeManager(); dtm.registerDocumentType(articleType); @@ -69,12 +71,12 @@ public class ProcessingUpdateTestCase { assertEquals(new StringFieldValue("body blah blah blah "), first.getFieldValue("title")); DocumentUpdate second = (DocumentUpdate) operations.get(1); - FieldUpdate firstUpd = second.getFieldUpdate(0); + FieldUpdate firstUpd = second.getFieldUpdate(bodyField); assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, firstUpd.getValueUpdate(0).getValueUpdateClassID()); assertEquals(new StringFieldValue("this is the updated body of the article, blahdi blahdi blahdi"), firstUpd.getValueUpdate(0) .getValue()); - FieldUpdate secondUpd = second.getFieldUpdate(1); + FieldUpdate secondUpd = second.getFieldUpdate(titleField); assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, secondUpd.getValueUpdate(0).getValueUpdateClassID()); assertEquals(new StringFieldValue("body blahdi blahdi blahdi "), secondUpd.getValueUpdate(0).getValue()); } diff --git a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java index 05a03480173..e6de3190156 100644 --- a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java +++ b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java @@ -328,16 +328,17 @@ public class SchemaMappingAndAccessesTest { Document doc = getDoc(); DocumentType type = doc.getDataType(); DocumentUpdate dud = new DocumentUpdate(type, new DocumentId("doc:map:test:1")); - FieldUpdate assignSingle = FieldUpdate.createAssign(type.getField("title"), new StringFieldValue("something")); + com.yahoo.document.Field title = type.getField("title"); + FieldUpdate assignSingle = FieldUpdate.createAssign(title, new StringFieldValue("something")); Map<String, String> fieldMap = new HashMap<>(); fieldMap.put("t", "title"); fieldMap.put("a", "artist"); ProxyDocumentUpdate pup = new ProxyDocumentUpdate(dud, fieldMap); pup.addFieldUpdate(assignSingle); - assertEquals(pup.getFieldUpdates(), dud.getFieldUpdates()); + assertEquals(pup.fieldUpdates().toString(), dud.fieldUpdates().toString()); assertEquals(pup.getDocumentType(), dud.getDocumentType()); - assertEquals(pup.getFieldUpdate(new com.yahoo.document.Field("title")).size(), 1); - assertEquals(pup.getFieldUpdate(0), dud.getFieldUpdate(0)); + assertEquals(pup.getFieldUpdate(title).size(), 1); + assertEquals(pup.getFieldUpdate(title), dud.fieldUpdates().iterator().next()); assertEquals(pup.getFieldUpdate("title"), dud.getFieldUpdate("title")); assertEquals(pup.getId(), dud.getId()); assertEquals(pup.getType(), dud.getType()); diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java index a2321e912e1..4905f3d9dad 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java @@ -51,13 +51,13 @@ public class DocumentScript { } public DocumentUpdate execute(AdapterFactory adapterFactory, DocumentUpdate update) { - for (FieldUpdate fieldUpdate : update.getFieldUpdates()) { + for (FieldUpdate fieldUpdate : update.fieldUpdates()) { requireThatFieldIsDeclaredInDocument(fieldUpdate.getField()); for (ValueUpdate<?> valueUpdate : fieldUpdate.getValueUpdates()) { removeAnyLinguisticsSpanTree(valueUpdate); } } - for (FieldPathUpdate fieldUpdate : update.getFieldPathUpdates()) { + for (FieldPathUpdate fieldUpdate : update.fieldPathUpdates()) { requireThatFieldIsDeclaredInDocument(fieldUpdate.getFieldPath().get(0).getFieldRef()); if (fieldUpdate instanceof AssignFieldPathUpdate) { removeAnyLinguisticsSpanTree(((AssignFieldPathUpdate)fieldUpdate).getFieldValue()); diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java index a47762bfbf3..cfaee10a07d 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java @@ -187,8 +187,8 @@ public class DocumentScriptTestCase { FieldPathUpdate executeWithUpdateAndExpectFieldPath(String fieldName, FieldPathUpdate updateIn) { DocumentUpdate update = executeWithUpdate(fieldName, updateIn); - assertEquals(1, update.getFieldPathUpdates().size()); - return update.getFieldPathUpdates().get(0); + assertEquals(1, update.fieldPathUpdates().size()); + return update.fieldPathUpdates().iterator().next(); } } @@ -229,10 +229,10 @@ public class DocumentScriptTestCase { StringFieldValue newTitleValue = new StringFieldValue("iron moose 4, moose with a vengeance"); DocumentUpdate update = f.executeWithUpdate("structfield", new AssignFieldPathUpdate(f.type, "structfield.title", newTitleValue)); - assertEquals(1, update.getFieldPathUpdates().size()); - assertEquals(0, update.getFieldUpdates().size()); - assertTrue(update.getFieldPathUpdates().get(0) instanceof AssignFieldPathUpdate); - AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)update.getFieldPathUpdates().get(0); + assertEquals(1, update.fieldPathUpdates().size()); + assertEquals(0, update.fieldUpdates().size()); + assertTrue(update.fieldPathUpdates().iterator().next() instanceof AssignFieldPathUpdate); + AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)update.fieldPathUpdates().iterator().next(); assertEquals("structfield.title", assignUpdate.getOriginalFieldPath()); assertEquals(newTitleValue, assignUpdate.getFieldValue()); } diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java index cef020cd828..5979672524d 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java @@ -13,7 +13,6 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.update.AssignValueUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.ValueUpdate; -import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import org.junit.Test; @@ -63,15 +62,15 @@ public class IndexingProcessorTestCase { assertTrue(output instanceof DocumentUpdate); DocumentUpdate docUpdate = (DocumentUpdate) output; - assertEquals(3, docUpdate.getFieldUpdates().size()); + assertEquals(3, docUpdate.fieldUpdates().size()); { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate(0); + FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("song"); assertEquals("song", fieldUpdate.getField().getName()); assertEquals(1, fieldUpdate.getValueUpdates().size()); ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); assertTrue(valueUpdate instanceof AssignValueUpdate); assertEquals(new StringFieldValue("isbnmarker"), valueUpdate.getValue()); - fieldUpdate = docUpdate.getFieldUpdate(1); + fieldUpdate = docUpdate.getFieldUpdate("title"); assertEquals("title", fieldUpdate.getField().getName()); assertEquals(1, fieldUpdate.getValueUpdates().size()); valueUpdate = fieldUpdate.getValueUpdate(0); @@ -80,14 +79,14 @@ public class IndexingProcessorTestCase { } { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate(1); + FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("title"); ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); assertEquals("title", fieldUpdate.getField().getName()); assertTrue(valueUpdate instanceof AssignValueUpdate); assertEquals(new StringFieldValue("69"), valueUpdate.getValue()); } { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate(2); + FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("isbn"); ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); assertEquals("isbn", fieldUpdate.getField().getName()); assertTrue(valueUpdate instanceof AssignValueUpdate); diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java index ad93942c1c0..54ced1321a0 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -13,6 +13,7 @@ import com.yahoo.document.update.ValueUpdate; import com.yahoo.io.GrowableByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -37,16 +38,18 @@ import java.util.Optional; * @see com.yahoo.document.update.FieldUpdate * @see com.yahoo.document.update.ValueUpdate */ +//TODO Vespa 7 Remove all deprecated methods and use a map to avoid quadratic scaling on insert/update/remove + public class DocumentUpdate extends DocumentOperation implements Iterable<FieldPathUpdate> { //see src/vespa/document/util/identifiableid.h public static final int CLASSID = 0x1000 + 6; private DocumentId docId; - private List<FieldUpdate> fieldUpdates; - private List<FieldPathUpdate> fieldPathUpdates; + private final List<FieldUpdate> fieldUpdates; + private final List<FieldPathUpdate> fieldPathUpdates; private DocumentType documentType; - private Optional<Boolean> createIfNonExistent = Optional.empty(); + private Boolean createIfNonExistent; /** * Creates a DocumentUpdate. @@ -55,7 +58,10 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param docType the document type that this update is valid for */ public DocumentUpdate(DocumentType docType, DocumentId docId) { - this(docType, docId, new ArrayList<FieldUpdate>()); + this.docId = docId; + this.documentType = docType; + this.fieldUpdates = new ArrayList<>(); + this.fieldPathUpdates = new ArrayList<>(); } /** @@ -79,13 +85,6 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP this(docType, new DocumentId(docId)); } - private DocumentUpdate(DocumentType docType, DocumentId docId, List<FieldUpdate> fieldUpdates) { - this.docId = docId; - this.documentType = docType; - this.fieldUpdates = fieldUpdates; - this.fieldPathUpdates = new ArrayList<>(); - } - public DocumentId getId() { return docId; } @@ -162,20 +161,42 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * Get an unmodifiable list of all field updates that this document update specifies. * * @return a list of all FieldUpdates in this DocumentUpdate + * @deprecated Use fieldUpdates() instead. */ + @Deprecated public List<FieldUpdate> getFieldUpdates() { return Collections.unmodifiableList(fieldUpdates); } /** + * Get an unmodifiable collection of all field updates that this document update specifies. + * + * @return a collection of all FieldUpdates in this DocumentUpdate + */ + public Collection<FieldUpdate> fieldUpdates() { + return Collections.unmodifiableCollection(fieldUpdates); + } + + /** * Get an unmodifiable list of all field path updates this document update specifies. * * @return Returns a list of all field path updates in this document update. + * @deprecated Use fieldPathUpdates() instead. */ + @Deprecated public List<FieldPathUpdate> getFieldPathUpdates() { return Collections.unmodifiableList(fieldPathUpdates); } + /** + * Get an unmodifiable collection of all field path updates that this document update specifies. + * + * @return a collection of all FieldPathUpdates in this DocumentUpdate + */ + public Collection<FieldPathUpdate> fieldPathUpdates() { + return Collections.unmodifiableCollection(fieldPathUpdates); + } + /** Returns the type of the document this updates * * @return The documentype of the document @@ -198,7 +219,9 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param index the index of the FieldUpdate to return * @return the FieldUpdate at the specified index * @throws IndexOutOfBoundsException if index is out of range + * @deprecated use getFieldUpdate(Field field) instead. */ + @Deprecated public FieldUpdate getFieldUpdate(int index) { return fieldUpdates.get(index); } @@ -210,9 +233,14 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param upd the FieldUpdate to be stored at the specified position * @return the FieldUpdate previously at the specified position * @throws IndexOutOfBoundsException if index is out of range + * @deprecated Use removeFieldUpdate/addFieldUpdate instead */ + @Deprecated public FieldUpdate setFieldUpdate(int index, FieldUpdate upd) { - return fieldUpdates.set(index, upd); + FieldUpdate old = fieldUpdates.get(index); + fieldUpdates.set(index, upd); + + return old; } /** @@ -222,7 +250,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return the update for the field, or null if that field has no update in this */ public FieldUpdate getFieldUpdate(Field field) { - return getFieldUpdate(field.getName()); + return getFieldUpdateById(field.getId()); } /** Removes all field updates from the list for field updates. */ @@ -237,8 +265,12 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return the update for the field, or null if that field has no update in this */ public FieldUpdate getFieldUpdate(String fieldName) { + Field field = documentType.getField(fieldName); + return field != null ? getFieldUpdate(field) : null; + } + private FieldUpdate getFieldUpdateById(Integer fieldId) { for (FieldUpdate fieldUpdate : fieldUpdates) { - if (fieldUpdate.getField().getName().equals(fieldName)) { + if (fieldUpdate.getField().getId() == fieldId) { return fieldUpdate; } } @@ -248,16 +280,24 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP /** * Assigns the field updates of this document update. * This document update receives ownership of the list - it can not be subsequently used - * by the caller. The list may not be unmodifiable. + * by the caller. Also note that there no assumptions can be made on the order of items + * after this call. They might have been joined if for the same field or reordered. * * @param fieldUpdates the new list of updates of this * @throws NullPointerException if the argument passed is null */ - public void setFieldUpdates(List<FieldUpdate> fieldUpdates) { + public void setFieldUpdates(Collection<FieldUpdate> fieldUpdates) { if (fieldUpdates == null) { throw new NullPointerException("The field updates of a document update can not be null"); } - this.fieldUpdates = fieldUpdates; + clearFieldUpdates(); + addFieldUpdates(fieldUpdates); + } + + public void addFieldUpdates(Collection<FieldUpdate> fieldUpdates) { + for (FieldUpdate fieldUpdate : fieldUpdates) { + addFieldUpdate(fieldUpdate); + } } /** @@ -279,12 +319,11 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * field. */ public DocumentUpdate addFieldUpdate(FieldUpdate update) { - String fieldName = update.getField().getName(); - if (!documentType.hasField(fieldName)) { - throw new IllegalArgumentException("Document type '" + documentType.getName() + "' does not have field '" + - fieldName + "'."); + int fieldId = update.getField().getId(); + if (documentType.getField(fieldId) == null) { + throw new IllegalArgumentException("Document type '" + documentType.getName() + "' does not have field '" + update.getField().getName() + "'."); } - FieldUpdate prevUpdate = getFieldUpdate(fieldName); + FieldUpdate prevUpdate = getFieldUpdateById(fieldId); if (prevUpdate != update) { if (prevUpdate != null) { prevUpdate.addAll(update); @@ -305,12 +344,6 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP return this; } - // TODO: Remove this when we figure out correct behaviour. - - public void addFieldUpdateNoCheck(FieldUpdate fieldUpdate) { - fieldUpdates.add(fieldUpdate); - } - /** * Adds all the field- and field path updates of the given document update to this. If the given update refers to a * different document or document type than this, this method throws an exception. @@ -329,9 +362,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP if (!documentType.equals(update.documentType)) { throw new IllegalArgumentException("Expected " + documentType + ", got " + update.documentType + "."); } - for (FieldUpdate fieldUpd : update.fieldUpdates) { - addFieldUpdate(fieldUpd); - } + addFieldUpdates(update.fieldUpdates()); for (FieldPathUpdate pathUpd : update.fieldPathUpdates) { addFieldPathUpdate(pathUpd); } @@ -343,9 +374,28 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param index the index of the FieldUpdate to remove * @return the FieldUpdate previously at the specified position * @throws IndexOutOfBoundsException if index is out of range + * @deprecated use removeFieldUpdate(Field field) instead. */ + @Deprecated public FieldUpdate removeFieldUpdate(int index) { - return fieldUpdates.remove(index); + FieldUpdate prev = getFieldUpdate(index); + fieldUpdates.remove(index); + return removeFieldUpdate(prev.getField()); + } + + public FieldUpdate removeFieldUpdate(Field field) { + for (Iterator<FieldUpdate> it = fieldUpdates.iterator(); it.hasNext();) { + FieldUpdate fieldUpdate = it.next(); + if (fieldUpdate.getField().equals(field)) { + it.remove(); + return fieldUpdate; + } + } + return null; } + + public FieldUpdate removeFieldUpdate(String fieldName) { + Field field = documentType.getField(fieldName); + return field != null ? removeFieldUpdate(field) : null; } /** @@ -398,23 +448,19 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP string.append(docId); string.append("': "); string.append("create-if-non-existent="); - string.append(createIfNonExistent.orElse(false)); + string.append(createIfNonExistent); string.append(": "); string.append("["); - for (Iterator<FieldUpdate> i = fieldUpdates.iterator(); i.hasNext();) { - FieldUpdate fieldUpdate = i.next(); - string.append(fieldUpdate); - if (i.hasNext()) { - string.append(", "); - } + for (FieldUpdate fieldUpdate : fieldUpdates) { + string.append(fieldUpdate).append(" "); } string.append("]"); if (fieldPathUpdates.size() > 0) { string.append(" [ "); for (FieldPathUpdate up : fieldPathUpdates) { - string.append(up.toString() + " "); + string.append(up.toString()).append(" "); } string.append(" ]"); } @@ -422,6 +468,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP return string.toString(); } + @Override public Iterator<FieldPathUpdate> iterator() { return fieldPathUpdates.iterator(); } @@ -443,7 +490,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param value Whether the document it updates should be created. */ public void setCreateIfNonExistent(boolean value) { - createIfNonExistent = Optional.of(value); + createIfNonExistent = value; } /** @@ -453,10 +500,10 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return Whether the document it updates should be created. */ public boolean getCreateIfNonExistent() { - return createIfNonExistent.orElse(false); + return createIfNonExistent != null && createIfNonExistent; } public Optional<Boolean> getOptionalCreateIfNonExistent() { - return createIfNonExistent; + return Optional.ofNullable(createIfNonExistent); } } diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index b9b273d691f..6adae27cadc 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.regex.Matcher; @@ -105,11 +106,11 @@ public class DocumentUpdateJsonSerializer } generator.writeObjectFieldStart("fields"); - for (FieldUpdate up : update.getFieldUpdates()) { + for (FieldUpdate up : update.fieldUpdates()) { up.serialize(this); } - update.getFieldPathUpdates().stream() + update.fieldPathUpdates().stream() .collect(Collectors.groupingBy(FieldPathUpdate::getFieldPath)) .forEach((fieldPath, fieldPathUpdates) -> wrapIOException(() -> write(fieldPath, fieldPathUpdates, generator))); @@ -120,7 +121,7 @@ public class DocumentUpdateJsonSerializer }); } - private void write(FieldPath fieldPath, List<FieldPathUpdate> fieldPathUpdates, JsonGenerator generator) throws IOException { + private void write(FieldPath fieldPath, Collection<FieldPathUpdate> fieldPathUpdates, JsonGenerator generator) throws IOException { generator.writeObjectFieldStart(fieldPath.toString()); for (FieldPathUpdate update : fieldPathUpdates) { diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java index a048ea349eb..3f885844987 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -472,7 +472,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple for (int i = 0; i < size; i++) { // TODO: Should use checked method, but doesn't work according to test now. - update.addFieldUpdateNoCheck(new FieldUpdate(this, update.getDocumentType(), serializationVersion)); + update.addFieldUpdate(new FieldUpdate(this, update.getDocumentType(), serializationVersion)); } } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java index 7bdc6fd5355..4f8a26d3777 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java @@ -29,7 +29,7 @@ public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer42 { for (int i = 0; i < size; i++) { // TODO: Should use checked method, but doesn't work according to test now. - update.addFieldUpdateNoCheck(new FieldUpdate(this, update.getDocumentType(), 8)); + update.addFieldUpdate(new FieldUpdate(this, update.getDocumentType(), 8)); } int sizeAndFlags = getInt(null); diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index 15319985591..a9f77cb5eb0 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -1,10 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.document.fieldpathupdate.FieldPathUpdate; -import com.yahoo.document.select.parser.ParseException; -import com.yahoo.document.serialization.*; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentDeserializerFactory; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.document.serialization.DocumentUpdateFlags; +import com.yahoo.document.serialization.DocumentUpdateWriter; import com.yahoo.document.update.AssignValueUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.ValueUpdate; @@ -259,8 +268,8 @@ public class DocumentUpdateTestCase { update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(1))); update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(2))); - assertEquals(1, update.getFieldUpdates().size()); - FieldUpdate fieldUpdate = update.getFieldUpdate(0); + assertEquals(1, update.fieldUpdates().size()); + FieldUpdate fieldUpdate = update.getFieldUpdate(field); assertNotNull(fieldUpdate); assertEquals(field, fieldUpdate.getField()); assertEquals(2, fieldUpdate.getValueUpdates().size()); @@ -342,13 +351,16 @@ public class DocumentUpdateTestCase { assertEquals(new DocumentId("doc:update:test"), upd.getId()); assertEquals(type, upd.getType()); - FieldUpdate serAssignFU = upd.getFieldUpdate(0); + FieldUpdate serAssignFU = upd.getFieldUpdate(type.getField("intfield")); assertEquals(type.getField("intfield"), serAssignFU.getField()); ValueUpdate serAssign = serAssignFU.getValueUpdate(0); assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, serAssign.getValueUpdateClassID()); assertEquals(new IntegerFieldValue(4), serAssign.getValue()); - FieldUpdate serAddFU = upd.getFieldUpdate(2); + ValueUpdate serArith = serAssignFU.getValueUpdate(1); + assertEquals(ValueUpdate.ValueUpdateClassID.ARITHMETIC, serArith.getValueUpdateClassID()); + + FieldUpdate serAddFU = upd.getFieldUpdate(type.getField("arrayoffloatfield")); assertEquals(type.getField("arrayoffloatfield"), serAddFU.getField()); ValueUpdate serAdd1 = serAddFU.getValueUpdate(0); assertEquals(ValueUpdate.ValueUpdateClassID.ADD, serAdd1.getValueUpdateClassID()); @@ -363,12 +375,7 @@ public class DocumentUpdateTestCase { FloatFieldValue addparam3 = (FloatFieldValue)serAdd3.getValue(); assertEquals(new FloatFieldValue(-1.00f), addparam3); - FieldUpdate arithFU = upd.getFieldUpdate(3); - assertEquals(type.getField("intfield"), serAssignFU.getField()); - ValueUpdate serArith = arithFU.getValueUpdate(0); - assertEquals(ValueUpdate.ValueUpdateClassID.ARITHMETIC, serArith.getValueUpdateClassID()); - - FieldUpdate wsetFU = upd.getFieldUpdate(4); + FieldUpdate wsetFU = upd.getFieldUpdate(type.getField("wsfield")); assertEquals(type.getField("wsfield"), wsetFU.getField()); assertEquals(2, wsetFU.size()); ValueUpdate mapUpd = wsetFU.getValueUpdate(0); @@ -420,8 +427,8 @@ public class DocumentUpdateTestCase { barUpdate.addFieldUpdate(barField); fooUpdate.addAll(barUpdate); - assertEquals(1, fooUpdate.getFieldUpdates().size()); - FieldUpdate fieldUpdate = fooUpdate.getFieldUpdate(0); + assertEquals(1, fooUpdate.fieldUpdates().size()); + FieldUpdate fieldUpdate = fooUpdate.getFieldUpdate(field); assertNotNull(fieldUpdate); assertEquals(field, fieldUpdate.getField()); assertEquals(2, fieldUpdate.getValueUpdates().size()); @@ -435,6 +442,32 @@ public class DocumentUpdateTestCase { } @Test + public void testGetAndRemoveByName() { + DocumentType docType = new DocumentType("my_type"); + Field my_int = new Field("my_int", DataType.INT); + Field your_int = new Field("your_int", DataType.INT); + docType.addField(my_int); + docType.addField(your_int); + DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:this:is:a:test")); + + update.addFieldUpdate(FieldUpdate.createAssign(my_int, new IntegerFieldValue(2))); + assertNull(update.getFieldUpdate("none-existing-field")); + assertNull(update.removeFieldUpdate("none-existing-field")); + assertNull(update.getFieldUpdate("your_int")); + assertEquals(new IntegerFieldValue(2), update.getFieldUpdate("my_int").getValueUpdate(0).getValue()); + assertNull(update.removeFieldUpdate("your_int")); + assertEquals(new IntegerFieldValue(2), update.removeFieldUpdate("my_int").getValueUpdate(0).getValue()); + assertNull(update.getFieldUpdate("my_int")); + + update.addFieldUpdate(FieldUpdate.createAssign(my_int, new IntegerFieldValue(2))); + assertNull(update.getFieldUpdate(your_int)); + assertEquals(new IntegerFieldValue(2), update.getFieldUpdate(my_int).getValueUpdate(0).getValue()); + assertNull(update.removeFieldUpdate(your_int)); + assertEquals(new IntegerFieldValue(2), update.removeFieldUpdate(my_int).getValueUpdate(0).getValue()); + assertNull(update.getFieldUpdate(my_int)); + } + + @Test public void testInstantiationAndEqualsHashCode() { DocumentType type = new DocumentType("doo"); DocumentUpdate d1 = new DocumentUpdate(type, new DocumentId("doc:this:is:a:test")); @@ -465,6 +498,7 @@ public class DocumentUpdateTestCase { } @Test + @SuppressWarnings("deprecation") public void testFieldUpdatesInDocUp() { DocumentType t1 = new DocumentType("doo"); Field f1 = new Field("field1", DataType.STRING); @@ -493,14 +527,6 @@ public class DocumentUpdateTestCase { assertSame(fu1, documentUpdate.getFieldUpdate(f1)); - assertSame(fu1, documentUpdate.getFieldUpdate(0)); - assertSame(fu2, documentUpdate.getFieldUpdate(1)); - - documentUpdate.setFieldUpdate(0, fu2); - documentUpdate.setFieldUpdate(1, fu1); - assertEquals(2, documentUpdate.size()); - assertSame(fu1, documentUpdate.getFieldUpdate(1)); - assertSame(fu2, documentUpdate.getFieldUpdate(0)); try { documentUpdate.setFieldUpdates(null); @@ -515,12 +541,12 @@ public class DocumentUpdateTestCase { documentUpdate.setFieldUpdates(fus); assertEquals(2, documentUpdate.size()); - assertSame(fu1, documentUpdate.getFieldUpdate(0)); - assertSame(fu2, documentUpdate.getFieldUpdate(1)); + assertSame(fu1, documentUpdate.getFieldUpdate(fu1.getField())); + assertSame(fu2, documentUpdate.getFieldUpdate(fu2.getField())); - documentUpdate.removeFieldUpdate(1); + documentUpdate.removeFieldUpdate(fu2.getField()); assertEquals(1, documentUpdate.size()); - assertSame(fu1, documentUpdate.getFieldUpdate(0)); + assertSame(fu1, documentUpdate.getFieldUpdate(fu1.getField())); documentUpdate.toString(); diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 1fd45cb07c4..32f63e6c0b3 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -292,8 +292,8 @@ public class JsonReaderTestCase { + "\"skuggsjaa\": {" + "\"assign\": { \"sandra\": \"person\"," + " \"cloud\": \"another person\"}}}}"); - assertEquals(1, put.getFieldUpdates().size()); - FieldUpdate fu = put.getFieldUpdate(0); + assertEquals(1, put.fieldUpdates().size()); + FieldUpdate fu = put.fieldUpdates().iterator().next(); assertEquals(1, fu.getValueUpdates().size()); ValueUpdate vu = fu.getValueUpdate(0); assertTrue(vu instanceof AssignValueUpdate); @@ -315,8 +315,8 @@ public class JsonReaderTestCase { + " \"fields\": { " + "\"skuggsjaa\": {" + "\"assign\": { }}}}"); - assertEquals(1, put.getFieldUpdates().size()); - FieldUpdate fu = put.getFieldUpdate(0); + assertEquals(1, put.fieldUpdates().size()); + FieldUpdate fu = put.fieldUpdates().iterator().next(); assertEquals(1, fu.getValueUpdates().size()); ValueUpdate vu = fu.getValueUpdate(0); assertTrue(vu instanceof AssignValueUpdate); diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java index ea954f0da40..dcdea0975ad 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java @@ -46,8 +46,8 @@ public class UriParserTestCase { DocumentUpdate upd = nextUpdate(it); assertNotNull(upd); - assertEquals(1, upd.getFieldUpdates().size()); - FieldUpdate fieldUpd = upd.getFieldUpdate(0); + assertEquals(1, upd.fieldUpdates().size()); + FieldUpdate fieldUpd = upd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_arr"), fieldUpd.getField()); assertEquals(1, fieldUpd.getValueUpdates().size()); diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java index 4c64f7c35cb..1aad59f4c56 100755 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java @@ -5,6 +5,7 @@ import com.yahoo.document.*; import com.yahoo.document.datatypes.*; import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; import com.yahoo.document.serialization.DeserializationException; import com.yahoo.document.update.AddValueUpdate; @@ -16,6 +17,7 @@ import org.junit.Before; import org.junit.Test; import java.io.ByteArrayInputStream; +import java.util.Iterator; import java.util.List; import static org.junit.Assert.*; @@ -687,103 +689,105 @@ public class VespaXMLReaderTestCase { DocumentUpdate docUpdate = op.getDocumentUpdate(); - assertEquals(20, docUpdate.getFieldPathUpdates().size()); + assertEquals(20, docUpdate.fieldPathUpdates().size()); + Iterator<FieldPathUpdate> updates = docUpdate.fieldPathUpdates().iterator(); { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(0); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("url", ass.getOriginalFieldPath()); assertEquals(new StringFieldValue("assignUrl"), ass.getNewValue()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(1); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("title", ass.getOriginalFieldPath()); assertEquals(new StringFieldValue("assignTitle"), ass.getNewValue()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(2); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("last_downloaded", ass.getOriginalFieldPath()); assertEquals("1", ass.getExpression()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(3); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("value_long", ass.getOriginalFieldPath()); assertEquals("2", ass.getExpression()); } + updates.next(); // Skip number 5 { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(5); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("stringarr", ass.getOriginalFieldPath()); assertEquals("[assignString1, assignString2]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(6); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("intarr", ass.getOriginalFieldPath()); assertEquals("[3, 4]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(7); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("longarr", ass.getOriginalFieldPath()); assertEquals("[5, 6]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(8); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("bytearr", ass.getOriginalFieldPath()); assertEquals("[7, 8]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(9); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("floatarr", ass.getOriginalFieldPath()); assertEquals("[9.0, 10.0]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(10); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("weightedsetint", ass.getOriginalFieldPath()); WeightedSet set = (WeightedSet)ass.getNewValue(); assertEquals(Integer.valueOf(11), set.get(new IntegerFieldValue(11))); assertEquals(Integer.valueOf(12), set.get(new IntegerFieldValue(12))); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(11); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("weightedsetstring", ass.getOriginalFieldPath()); WeightedSet set = (WeightedSet)ass.getNewValue(); assertEquals(Integer.valueOf(13), set.get(new StringFieldValue("assign13"))); assertEquals(Integer.valueOf(14), set.get(new StringFieldValue("assign14"))); } { - AddFieldPathUpdate ass = (AddFieldPathUpdate)docUpdate.getFieldPathUpdates().get(12); + AddFieldPathUpdate ass = (AddFieldPathUpdate)updates.next(); assertEquals("stringarr", ass.getOriginalFieldPath()); assertEquals("[addString1, addString2]", ass.getNewValues().toString()); } { - AddFieldPathUpdate ass = (AddFieldPathUpdate)docUpdate.getFieldPathUpdates().get(13); + AddFieldPathUpdate ass = (AddFieldPathUpdate)updates.next(); assertEquals("longarr", ass.getOriginalFieldPath()); assertEquals("[5]", ass.getNewValues().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(14); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("weightedsetint{13}", ass.getOriginalFieldPath()); assertEquals("13", ass.getExpression()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(15); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("weightedsetint{14}", ass.getOriginalFieldPath()); assertEquals("14", ass.getExpression()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(16); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("weightedsetstring{add13}", ass.getOriginalFieldPath()); assertEquals("1", ass.getExpression()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(17); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); assertEquals("weightedsetstring{assign13}", ass.getOriginalFieldPath()); assertEquals("130", ass.getExpression()); } { - RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)docUpdate.getFieldPathUpdates().get(18); + RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)updates.next(); assertEquals("weightedsetstring{assign14}", ass.getOriginalFieldPath()); } { - RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)docUpdate.getFieldPathUpdates().get(19); + RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)updates.next(); assertEquals("bytearr", ass.getOriginalFieldPath()); } Document doc = new Document(manager.getDocumentType("news"), new DocumentId("doc:test:test:test")); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java index 939879c5448..1f70cfdaaf0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java @@ -12,11 +12,12 @@ import com.yahoo.document.DocumentId; public class DocumentIdResponse extends Response { /** The document id of this response, if any */ - private DocumentId documentId = null; + private final DocumentId documentId; /** Creates a successful response */ public DocumentIdResponse(long requestId) { super(requestId); + documentId = null; } /** @@ -37,6 +38,7 @@ public class DocumentIdResponse extends Response { */ public DocumentIdResponse(long requestId, String textMessage, boolean success) { super(requestId, textMessage, success); + documentId = null; } /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java index dab4f2e7855..983b4a0d5b2 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java @@ -2,7 +2,6 @@ package com.yahoo.documentapi; import com.yahoo.document.Document; -import com.yahoo.component.Version; /** * The asynchronous response to a document put or get operation. @@ -13,11 +12,12 @@ import com.yahoo.component.Version; public class DocumentResponse extends Response { /** The document of this response, if any */ - private Document document = null; + private final Document document; /** Creates a successful response */ public DocumentResponse(long requestId) { super(requestId); + document = null; } /** @@ -38,6 +38,7 @@ public class DocumentResponse extends Response { */ public DocumentResponse(long requestId, String textMessage, boolean success) { super(requestId, textMessage, success); + document = null; } /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Response.java b/documentapi/src/main/java/com/yahoo/documentapi/Response.java index 39b77c40e1d..4d8cb93d1d0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Response.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Response.java @@ -11,9 +11,9 @@ package com.yahoo.documentapi; */ public class Response { - private long requestId; - private String textMessage = null; - private boolean success = true; + private final long requestId; + private final String textMessage; + private final boolean success; /** Creates a successful response containing no information */ public Response(long requestId) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java index d365407f407..041c8187a97 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java @@ -31,7 +31,6 @@ public class LocalAsyncSession implements AsyncSession { private final List<Response> responses = new LinkedList<>(); private final ResponseHandler handler; - private final LocalDocumentAccess access; private final SyncSession syncSession; private long requestId = 0; private Random random = new Random(); @@ -42,7 +41,6 @@ public class LocalAsyncSession implements AsyncSession { } public LocalAsyncSession(AsyncParameters params, LocalDocumentAccess access) { - this.access = access; this.handler = params.getResponseHandler(); random.setSeed(System.currentTimeMillis()); syncSession = access.createSyncSession(new SyncParameters.Builder().build()); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java index c37e871005c..202929130c7 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java @@ -3,10 +3,21 @@ package com.yahoo.documentapi.local; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; -import com.yahoo.documentapi.*; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.SubscriptionParameters; +import com.yahoo.documentapi.SubscriptionSession; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; -import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * The main class of the local implementation of the document api @@ -15,7 +26,7 @@ import java.util.Map; */ public class LocalDocumentAccess extends DocumentAccess { - Map<DocumentId, Document> documents = new LinkedHashMap<DocumentId, Document>(); + Map<DocumentId, Document> documents = new ConcurrentHashMap<>(); public LocalDocumentAccess(DocumentAccessParams params) { super(params); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java index b1776b8c96d..45df2015da4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java @@ -5,13 +5,10 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.TestAndSetCondition; import com.yahoo.documentapi.Response; -import com.yahoo.documentapi.Result; import com.yahoo.documentapi.SyncSession; -import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; /** * @author bratseth diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java index c18bfc7597a..ac2db526689 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java @@ -7,9 +7,22 @@ import com.yahoo.documentapi.DocumentAccessParams; import com.yahoo.documentapi.SyncParameters; import com.yahoo.documentapi.SyncSession; import com.yahoo.documentapi.local.LocalDocumentAccess; -import com.yahoo.documentapi.messagebus.protocol.*; -import com.yahoo.messagebus.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageHandler; +import com.yahoo.messagebus.Protocol; +import com.yahoo.messagebus.RPCMessageBus; +import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java index bdac2112263..bb84b6f0104 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java @@ -2,7 +2,10 @@ package com.yahoo.documentapi.messagebus.test; import com.yahoo.document.select.parser.ParseException; -import com.yahoo.documentapi.*; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; @@ -19,10 +22,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** diff --git a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java index 6160c9deca6..2b56d9c628d 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java @@ -6,7 +6,15 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; -import com.yahoo.documentapi.*; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentResponse; +import com.yahoo.documentapi.Response; +import com.yahoo.documentapi.ResponseHandler; +import com.yahoo.documentapi.Result; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; import org.junit.Test; import java.util.ArrayList; @@ -16,7 +24,10 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * These tests should work with all implementations (who choose to implement these features) To test a certain diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java index 43ccd6d48a8..35bb70c2a88 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java @@ -61,7 +61,7 @@ public class SimpleAdapterFactory implements AdapterFactory { throw new IllegalArgumentException("Exception during handling of update '" + fieldUpd + "' to field '" + fieldUpd.getFieldPath() + "'", e); } } - for (FieldUpdate fieldUpd : upd.getFieldUpdates()) { + for (FieldUpdate fieldUpd : upd.fieldUpdates()) { Field field = fieldUpd.getField(); for (ValueUpdate valueUpd : fieldUpd.getValueUpdates()) { try { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java index 459f3ce827c..3c5eb9ea1c5 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java @@ -29,8 +29,8 @@ public class DocumentToPathUpdateTestCase { DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldPathUpdates().size()); - assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); + assertEquals(1, docUpd.fieldPathUpdates().size()); + assertNotNull(upd = docUpd.fieldPathUpdates().iterator().next()); assertTrue(upd instanceof AssignFieldPathUpdate); assertEquals("my_int", upd.getOriginalFieldPath()); @@ -49,8 +49,8 @@ public class DocumentToPathUpdateTestCase { DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldPathUpdates().size()); - assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); + assertEquals(1, docUpd.fieldPathUpdates().size()); + assertNotNull(upd = docUpd.fieldPathUpdates().iterator().next()); assertTrue(upd instanceof AssignFieldPathUpdate); assertEquals("my_str", upd.getOriginalFieldPath()); @@ -77,8 +77,8 @@ public class DocumentToPathUpdateTestCase { DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldPathUpdates().size()); - assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); + assertEquals(1, docUpd.fieldPathUpdates().size()); + assertNotNull(upd = docUpd.fieldPathUpdates().iterator().next()); assertTrue(upd instanceof AssignFieldPathUpdate); assertEquals("a", upd.getOriginalFieldPath()); @@ -103,8 +103,8 @@ public class DocumentToPathUpdateTestCase { DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldPathUpdates().size()); - assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); + assertEquals(1, docUpd.fieldPathUpdates().size()); + assertNotNull(upd = docUpd.fieldPathUpdates().iterator().next()); assertTrue(upd instanceof AssignFieldPathUpdate); assertEquals("a.b", upd.getOriginalFieldPath()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java index de090163b7b..83947b5f64d 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java @@ -40,8 +40,8 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); - assertEquals("my_int", docUpd.getFieldUpdate(0).getField().getName()); + assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals("my_int", docUpd.fieldUpdates().iterator().next().getField().getName()); } @Test @@ -56,9 +56,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_int"), fieldUpd.getField()); @@ -80,9 +80,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_int"), fieldUpd.getField()); @@ -103,9 +103,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_str"), fieldUpd.getField()); @@ -136,9 +136,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("a"), fieldUpd.getField()); @@ -166,9 +166,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_arr"), fieldUpd.getField()); @@ -196,9 +196,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_arr"), fieldUpd.getField()); @@ -231,9 +231,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("a"), fieldUpd.getField()); @@ -261,9 +261,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_wset"), fieldUpd.getField()); @@ -294,9 +294,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_wset"), fieldUpd.getField()); @@ -326,9 +326,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals(1, docUpd.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_wset"), fieldUpd.getField()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java index bdb8dbedf78..dc8ffcd8d10 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java @@ -32,10 +32,10 @@ public class DocumentUpdateTestCase { docUpdate = Expression.execute(Expression.fromString("input my_str | for_each { to_pos } | index my_pos"), docUpdate); assertNotNull(docUpdate); - assertEquals(0, docUpdate.getFieldPathUpdates().size()); - assertEquals(1, docUpdate.getFieldUpdates().size()); + assertEquals(0, docUpdate.fieldPathUpdates().size()); + assertEquals(1, docUpdate.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpdate.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpdate.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_pos"), fieldUpd.getField()); assertEquals(1, fieldUpd.getValueUpdates().size()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java index 033034fed1f..63a2cc66a97 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java @@ -72,10 +72,10 @@ public class GuardTestCase { docUpdate.addFieldUpdate(FieldUpdate.createAssign(docType.getField("my_str"), new StringFieldValue("69"))); assertNotNull(docUpdate = Expression.execute(Expression.fromString("guard { input my_str | to_int | attribute my_lng }"), docUpdate)); - assertEquals(0, docUpdate.getFieldPathUpdates().size()); - assertEquals(1, docUpdate.getFieldUpdates().size()); + assertEquals(0, docUpdate.fieldPathUpdates().size()); + assertEquals(1, docUpdate.fieldUpdates().size()); - FieldUpdate fieldUpd = docUpdate.getFieldUpdate(0); + FieldUpdate fieldUpd = docUpdate.fieldUpdates().iterator().next(); assertNotNull(fieldUpd); assertEquals(docType.getField("my_lng"), fieldUpd.getField()); assertEquals(1, fieldUpd.getValueUpdates().size()); diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java index 6208bc97077..f543e1b5c8d 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java @@ -4,8 +4,16 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.IntermediateSession; +import com.yahoo.messagebus.IntermediateSessionParams; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageHandler; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Result; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; @@ -19,7 +27,6 @@ public class SharedIntermediateSession extends AbstractResource private static final Logger log = Logger.getLogger(SharedIntermediateSession.class.getName()); private final AtomicReference<MessageHandler> msgHandler = new AtomicReference<>(); - private final SharedMessageBus mbus; private final IntermediateSession session; private final ResourceReference mbusReference; @@ -27,7 +34,6 @@ public class SharedIntermediateSession extends AbstractResource if (params.getReplyHandler() != null) { throw new IllegalArgumentException("Reply handler must be null."); } - this.mbus = mbus; this.msgHandler.set(params.getMessageHandler()); this.session = mbus.messageBus().createIntermediateSession(params.setReplyHandler(this) .setMessageHandler(this)); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java index fef5f8afa2d..75534605153 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus; -import com.yahoo.log.LogLevel; - import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java index 6a99614fa4e..b804c776969 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java @@ -19,7 +19,7 @@ public class Sequencer implements MessageHandler, ReplyHandler { private final AtomicBoolean destroyed = new AtomicBoolean(false); private final MessageHandler sender; - private final Map<Long, Queue<Message>> seqMap = new HashMap<Long, Queue<Message>>(); + private final Map<Long, Queue<Message>> seqMap = new HashMap<>(); /** * Constructs a new sequencer on top of the given async sender. diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 64958554f53..476902e400c 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -91,6 +91,7 @@ <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> + <version>2.23.0</version> <scope>test</scope> </dependency> <dependency> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 7b484dfc481..ec911cc5600 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import java.net.URI; import java.util.ArrayList; @@ -24,12 +22,6 @@ public class ConfigServerInfo { private final Map<String, URI> configServerURIs; private final AthenzService configServerIdentity; - // TODO: Remove - public ConfigServerInfo(ConfigServerConfig config) { - this(config.loadBalancerHost(), config.hosts(), config.scheme(), config.port(), - (AthenzService) AthenzIdentities.from(config.configserverAthenzIdentity())); - } - public ConfigServerInfo(String loadBalancerHostName, List<String> configServerHostNames, String scheme, int port, AthenzService configServerAthenzIdentity) { this.configServerHostNames = configServerHostNames; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java deleted file mode 100644 index aaadd3cb24e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesImpl; - -import java.net.URI; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Various utilities for getting values from node-admin's environment. Immutable. - * - * @author Øyvind Bakksjø - * @author hmusum - */ -public class Environment { - private final ConfigServerInfo configServerInfo; - private final String environment; - private final String region; - private final String system; - private final String cloud; - private final String parentHostHostname; - private final IPAddresses ipAddresses; - private final PathResolver pathResolver; - private final List<String> logstashNodes; - private final NodeType nodeType; - private final ContainerEnvironmentResolver containerEnvironmentResolver; - private final String certificateDnsSuffix; - private final URI ztsUri; - private final AthenzService nodeAthenzIdentity; - private final boolean nodeAgentCertEnabled; - private final Path trustStorePath; - private final DockerNetworking dockerNetworking; - - private Environment(ConfigServerInfo configServerInfo, - Path trustStorePath, - String environment, - String region, - String system, - String cloud, - String parentHostHostname, - IPAddresses ipAddresses, - PathResolver pathResolver, - List<String> logstashNodes, - NodeType nodeType, - ContainerEnvironmentResolver containerEnvironmentResolver, - String certificateDnsSuffix, - URI ztsUri, - AthenzService nodeAthenzIdentity, - boolean nodeAgentCertEnabled, - DockerNetworking dockerNetworking) { - this.configServerInfo = Objects.requireNonNull(configServerInfo, "configServerConfig cannot be null"); - this.environment = Objects.requireNonNull(environment, "environment cannot be null");; - this.region = Objects.requireNonNull(region, "region cannot be null");; - this.system = Objects.requireNonNull(system, "system cannot be null");; - this.cloud = Objects.requireNonNull(cloud, "cloud cannot be null"); - this.parentHostHostname = parentHostHostname; - this.ipAddresses = ipAddresses; - this.pathResolver = pathResolver; - this.logstashNodes = logstashNodes; - this.nodeType = nodeType; - this.containerEnvironmentResolver = containerEnvironmentResolver; - this.certificateDnsSuffix = certificateDnsSuffix; - this.ztsUri = ztsUri; - this.nodeAthenzIdentity = nodeAthenzIdentity; - this.nodeAgentCertEnabled = nodeAgentCertEnabled; - this.trustStorePath = trustStorePath; - this.dockerNetworking = Objects.requireNonNull(dockerNetworking, "dockerNetworking cannot be null"); - } - - public List<String> getConfigServerHostNames() { return configServerInfo.getConfigServerHostNames(); } - - public String getEnvironment() { return environment; } - - public String getRegion() { - return region; - } - - public String getSystem() { - return system; - } - - public String getCloud() { return cloud; } - - public String getParentHostHostname() { - return parentHostHostname; - } - - public String getZone() { - return getEnvironment() + "." + getRegion(); - } - - public IPAddresses getIpAddresses() { - return ipAddresses; - } - - public PathResolver getPathResolver() { - return pathResolver; - } - - /** - * Translates an absolute path in node agent container to an absolute path in node admin container. - * @param containerName name of the node agent container - * @param pathInNode absolute path in that container - * @return the absolute path in node admin container pointing at the same inode - */ - public Path pathInNodeAdminFromPathInNode(ContainerName containerName, Path pathInNode) { - if (! pathInNode.isAbsolute()) { - throw new IllegalArgumentException("The specified path in node was not absolute: " + pathInNode); - } - - return pathResolver.getApplicationStoragePathForNodeAdmin() - .resolve(containerName.asString()) - .resolve(PathResolver.ROOT.relativize(pathInNode)); - } - - /** - * Translates an absolute path in node agent container to an absolute path in host. - * @param containerName name of the node agent container - * @param pathInNode absolute path in that container - * @return the absolute path in host pointing at the same inode - */ - public Path pathInHostFromPathInNode(ContainerName containerName, Path pathInNode) { - if (! pathInNode.isAbsolute()) { - throw new IllegalArgumentException("The specified path in node was not absolute: " + pathInNode); - } - - return pathResolver.getApplicationStoragePathForHost() - .resolve(containerName.asString()) - .resolve(PathResolver.ROOT.relativize(pathInNode)); - } - - public Path pathInNodeUnderVespaHome(String relativePath) { - return pathResolver.getVespaHomePathForContainer() - .resolve(relativePath); - } - - public List<String> getLogstashNodes() { - return logstashNodes; - } - - public NodeType getNodeType() { return nodeType; } - - public ContainerEnvironmentResolver getContainerEnvironmentResolver() { - return containerEnvironmentResolver; - } - - public Path getTrustStorePath() { - return trustStorePath; - } - - public AthenzService getConfigserverAthenzIdentity() { - return configServerInfo.getConfigServerIdentity(); - } - - public AthenzService getNodeAthenzIdentity() { - return nodeAthenzIdentity; - } - - public String getCertificateDnsSuffix() { - return certificateDnsSuffix; - } - - public URI getZtsUri() { - return ztsUri; - } - - public URI getConfigserverLoadBalancerEndpoint() { - return configServerInfo.getLoadBalancerEndpoint(); - } - - public boolean isNodeAgentCertEnabled() { - return nodeAgentCertEnabled; - } - - public DockerNetworking getDockerNetworking() { - return dockerNetworking; - } - - public static class Builder { - private ConfigServerInfo configServerInfo; - private String environment; - private String region; - private String system; - private String cloud; - private String parentHostHostname; - private IPAddresses ipAddresses; - private PathResolver pathResolver; - private List<String> logstashNodes = Collections.emptyList(); - private NodeType nodeType = NodeType.tenant; - private ContainerEnvironmentResolver containerEnvironmentResolver; - private String certificateDnsSuffix; - private URI ztsUri; - private AthenzService nodeAthenzIdentity; - private boolean nodeAgentCertEnabled; - private Path trustStorePath; - private DockerNetworking dockerNetworking; - - public Builder configServerConfig(ConfigServerConfig configServerConfig) { - this.configServerInfo = new ConfigServerInfo(configServerConfig); - return this; - } - - public Builder configServerInfo(ConfigServerInfo configServerInfo) { - this.configServerInfo = configServerInfo; - return this; - } - - public Builder environment(String environment) { - this.environment = environment; - return this; - } - - public Builder region(String region) { - this.region = region; - return this; - } - - public Builder system(String system) { - this.system = system; - return this; - } - - public Builder cloud(String cloud) { - this.cloud = cloud; - return this; - } - - public Builder parentHostHostname(String parentHostHostname) { - this.parentHostHostname = parentHostHostname; - return this; - } - - public Builder ipAddresses(IPAddresses ipAddresses) { - this.ipAddresses = ipAddresses; - return this; - } - - public Builder pathResolver(PathResolver pathResolver) { - this.pathResolver = pathResolver; - return this; - } - - public Builder containerEnvironmentResolver(ContainerEnvironmentResolver containerEnvironmentResolver) { - this.containerEnvironmentResolver = containerEnvironmentResolver; - return this; - } - - public Builder logstashNodes(List<String> hosts) { - this.logstashNodes = hosts; - return this; - } - - public Builder nodeType(NodeType nodeType) { - this.nodeType = nodeType; - return this; - } - - public Builder certificateDnsSuffix(String certificateDnsSuffix) { - this.certificateDnsSuffix = certificateDnsSuffix; - return this; - } - - public Builder ztsUri(URI ztsUri) { - this.ztsUri = ztsUri; - return this; - } - - public Builder nodeAthenzIdentity(AthenzService nodeAthenzIdentity) { - this.nodeAthenzIdentity = nodeAthenzIdentity; - return this; - } - - public Builder enableNodeAgentCert(boolean nodeAgentCertEnabled) { - this.nodeAgentCertEnabled = nodeAgentCertEnabled; - return this; - } - - public Builder trustStorePath(Path trustStorePath) { - this.trustStorePath = trustStorePath; - return this; - } - - public Builder dockerNetworking(DockerNetworking dockerNetworking) { - this.dockerNetworking = dockerNetworking; - return this; - } - - public Environment build() { - return new Environment(configServerInfo, - trustStorePath, - environment, - region, - system, - cloud, - parentHostHostname, - Optional.ofNullable(ipAddresses).orElseGet(IPAddressesImpl::new), - Optional.ofNullable(pathResolver).orElseGet(PathResolver::new), - logstashNodes, - nodeType, - Optional.ofNullable(containerEnvironmentResolver).orElseGet(() -> node -> ""), - certificateDnsSuffix, - ztsUri, - nodeAthenzIdentity, - nodeAgentCertEnabled, - dockerNetworking); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java deleted file mode 100644 index 433fae0e551..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.vespa.defaults.Defaults; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * @author freva - */ -public class PathResolver { - public static final Path ROOT = Paths.get("/"); - public static final Path DEFAULT_HOST_ROOT = Paths.get("/host"); - public static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); - - private final Path hostRoot; - private final Path vespaHomePathForContainer; - - private final Path applicationStoragePathForNodeAdmin; - private final Path applicationStoragePathForHost; - - /** - * @param hostRoot the absolute path to the root of the host's file system - * @param vespaHomeForContainer the absolute path of Vespa home in the mount namespace of any - * and all Docker containers managed by Node Admin. - */ - public PathResolver(Path hostRoot, Path vespaHomeForContainer) { - if (!hostRoot.isAbsolute()) { - throw new IllegalArgumentException("Path to root of host file system is not absolute: " + - hostRoot); - } - this.hostRoot = hostRoot; - - if (!vespaHomeForContainer.isAbsolute()) { - throw new IllegalArgumentException("Path to Vespa home is not absolute: " + vespaHomeForContainer); - } - this.vespaHomePathForContainer = vespaHomeForContainer; - - this.applicationStoragePathForNodeAdmin = hostRoot.resolve(RELATIVE_APPLICATION_STORAGE_PATH); - this.applicationStoragePathForHost = ROOT.resolve(RELATIVE_APPLICATION_STORAGE_PATH); - } - - public PathResolver() { - this(DEFAULT_HOST_ROOT, Paths.get(Defaults.getDefaults().vespaHome())); - } - - /** For testing */ - public PathResolver(Path vespaHomePathForContainer, Path applicationStoragePathForNodeAdmin, Path applicationStoragePathForHost) { - this.hostRoot = DEFAULT_HOST_ROOT; - this.vespaHomePathForContainer = vespaHomePathForContainer; - this.applicationStoragePathForNodeAdmin = applicationStoragePathForNodeAdmin; - this.applicationStoragePathForHost = applicationStoragePathForHost; - } - - /** - * Returns the absolute path of the Vespa home directory in any Docker container mount namespace. - * - * It's a limitation of current implementation that all containers MUST have the same Vespa - * home directory path. - */ - public Path getVespaHomePathForContainer() { - return vespaHomePathForContainer; - } - - /** Returns the absolute path to the container storage directory for the node admin (this process). */ - public Path getApplicationStoragePathForNodeAdmin() { - return applicationStoragePathForNodeAdmin; - } - - /** Returns the absolute path to the container storage directory for the host. */ - public Path getApplicationStoragePathForHost() { - return applicationStoragePathForHost; - } - - /** - * Returns the absolute path to the directory which is the root directory of the host - * file system. - */ - public Path getPathToRootOfHost() { - return hostRoot; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java index 9a94231437d..91ff159ac41 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java @@ -130,6 +130,6 @@ public class NodeAttributes { wantToDeprovision.map(depr -> "wantToDeprovision=" + depr)) .filter(Optional::isPresent) .map(Optional::get) - .collect(Collectors.joining(", ", "NodeAttributes{", "}")); + .collect(Collectors.joining(", ", "{", "}")); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java index b5d41b7fbb4..ba4e7ab7b7c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/Orchestrator.java @@ -9,23 +9,28 @@ import java.util.List; * @author bakksjo */ public interface Orchestrator { + /** - * Invokes orchestrator suspend of a host. - * @throws OrchestratorException if suspend was denied. + * Suspends a host. + * + * @throws OrchestratorException if suspend was denied * @throws OrchestratorNotFoundException if host is unknown to the orchestrator */ void suspend(String hostName); /** - * Invokes orchestrator resume of a host. + * Resumes a host. + * * @throws OrchestratorException if resume was denied * @throws OrchestratorNotFoundException if host is unknown to the orchestrator */ void resume(String hostName); /** - * Invokes orchestrator suspend hosts. - * @throws OrchestratorException if batch suspend was denied. + * Suspends a list of nodes on a parent. + * + * @throws OrchestratorException if batch suspend was denied */ void suspend(String parentHostName, List<String> hostNames); + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 7a83f00e297..41fe67f1996 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -304,9 +304,9 @@ public class DockerOperationsImpl implements DockerOperations { context.pathInNodeUnderVespaHome("tmp"), context.pathInNodeUnderVespaHome("var/container-data"))); - if (context.nodeType() == NodeType.proxyhost) + if (context.nodeType() == NodeType.proxy) paths.add(context.pathInNodeUnderVespaHome("var/vespa-hosted/routing")); - if (context.nodeType() == NodeType.host) + if (context.nodeType() == NodeType.tenant) paths.add(varLibSia); paths.forEach(path -> command.withVolume(context.pathOnHostFromPathInNode(path), path)); @@ -316,18 +316,18 @@ public class DockerOperationsImpl implements DockerOperations { if (isInfrastructureHost(context.nodeType())) command.withSharedVolume(varLibSia, varLibSia); - if (context.nodeType() == NodeType.proxyhost || context.nodeType() == NodeType.controllerhost) + if (context.nodeType() == NodeType.proxy || context.nodeType() == NodeType.controller) command.withSharedVolume(Paths.get("/opt/yahoo/share/ssl/certs"), Paths.get("/opt/yahoo/share/ssl/certs")); - if (context.nodeType() == NodeType.host) + if (context.nodeType() == NodeType.tenant) command.withSharedVolume(Paths.get("/var/zpe"), context.pathInNodeUnderVespaHome("var/zpe")); } /** Returns whether given nodeType is a Docker host for infrastructure nodes */ private static boolean isInfrastructureHost(NodeType nodeType) { - return nodeType == NodeType.confighost || - nodeType == NodeType.proxyhost || - nodeType == NodeType.controllerhost; + return nodeType == NodeType.config || + nodeType == NodeType.proxy || + nodeType == NodeType.controller; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java deleted file mode 100644 index ce751548f75..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.logging; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.component.Environment; - -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * @author mortent - */ -public class FilebeatConfigProvider { - - private static final String TENANT_FIELD = "%%TENANT%%"; - private static final String APPLICATION_FIELD = "%%APPLICATION%%"; - private static final String INSTANCE_FIELD = "%%INSTANCE%%"; - private static final String ENVIRONMENT_FIELD = "%%ENVIRONMENT%%"; - private static final String REGION_FIELD = "%%REGION%%"; - private static final String FILEBEAT_SPOOL_SIZE_FIELD = "%%FILEBEAT_SPOOL_SIZE%%"; - private static final String LOGSTASH_HOSTS_FIELD = "%%LOGSTASH_HOSTS%%"; - private static final String LOGSTASH_WORKERS_FIELD = "%%LOGSTASH_WORKERS%%"; - private static final String LOGSTASH_BULK_MAX_SIZE_FIELD = "%%LOGSTASH_BULK_MAX_SIZE%%"; - - private static final int logstashWorkers = 3; - private static final int logstashBulkMaxSize = 2048; - private final Environment environment; - - public FilebeatConfigProvider(Environment environment) { - this.environment = environment; - } - - public Optional<String> getConfig(NodeAgentContext context, NodeSpec node) { - - if (environment.getLogstashNodes().size() == 0 || !node.getOwner().isPresent()) { - return Optional.empty(); - } - NodeSpec.Owner owner = node.getOwner().get(); - int spoolSize = environment.getLogstashNodes().size() * logstashWorkers * logstashBulkMaxSize; - String logstashNodeString = environment.getLogstashNodes().stream() - .map(this::addQuotes) - .collect(Collectors.joining(",")); - return Optional.of(getTemplate(context) - .replaceAll(ENVIRONMENT_FIELD, environment.getEnvironment()) - .replaceAll(REGION_FIELD, environment.getRegion()) - .replaceAll(FILEBEAT_SPOOL_SIZE_FIELD, Integer.toString(spoolSize)) - .replaceAll(LOGSTASH_HOSTS_FIELD, logstashNodeString) - .replaceAll(LOGSTASH_WORKERS_FIELD, Integer.toString(logstashWorkers)) - .replaceAll(LOGSTASH_BULK_MAX_SIZE_FIELD, Integer.toString(logstashBulkMaxSize)) - .replaceAll(TENANT_FIELD, owner.getTenant()) - .replaceAll(APPLICATION_FIELD, owner.getApplication()) - .replaceAll(INSTANCE_FIELD, owner.getInstance())); - } - - private String addQuotes(String logstashNode) { - return logstashNode.startsWith("\"") - ? logstashNode - : String.format("\"%s\"", logstashNode); - } - - private String getTemplate(NodeAgentContext context) { - return "################### Filebeat Configuration Example #########################\n" + - "\n" + - "############################# Filebeat ######################################\n" + - "filebeat:\n" + - " # List of prospectors to fetch data.\n" + - " prospectors:\n" + - "\n" + - " # vespa\n" + - " - paths:\n" + - " - " + context.pathInNodeUnderVespaHome("logs/vespa/vespa.log") + "\n" + - " exclude_files: [\".gz$\"]\n" + - " document_type: vespa\n" + - " fields:\n" + - " HV-tenant: %%TENANT%%\n" + - " HV-application: %%APPLICATION%%\n" + - " HV-instance: %%INSTANCE%%\n" + - " HV-region: %%REGION%%\n" + - " HV-environment: %%ENVIRONMENT%%\n" + - " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" + - " fields_under_root: true\n" + - " close_older: 20m\n" + - " force_close_files: true\n" + - "\n" + - " # vespa qrs\n" + - " - paths:\n" + - " - " + context.pathInNodeUnderVespaHome("logs/vespa/qrs/QueryAccessLog.*.*") + "\n" + - " exclude_files: [\".gz$\"]\n" + - " exclude_lines: [\"reserved-for-internal-use/feedapi\"]\n" + - " document_type: vespa-qrs\n" + - " fields:\n" + - " HV-tenant: %%TENANT%%\n" + - " HV-application: %%APPLICATION%%\n" + - " HV-instance: %%INSTANCE%%\n" + - " HV-region: %%REGION%%\n" + - " HV-environment: %%ENVIRONMENT%%\n" + - " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" + - " fields_under_root: true\n" + - " close_older: 20m\n" + - " force_close_files: true\n" + - "\n" + - " # General filebeat configuration options\n" + - " #\n" + - " # Event count spool threshold - forces network flush if exceeded\n" + - " spool_size: %%FILEBEAT_SPOOL_SIZE%%\n" + - "\n" + - " # Defines how often the spooler is flushed. After idle_timeout the spooler is\n" + - " # Flush even though spool_size is not reached.\n" + - " #idle_timeout: 5s\n" + - " publish_async: false\n" + - "\n" + - " # Name of the registry file. Per default it is put in the current working\n" + - " # directory. In case the working directory is changed after when running\n" + - " # filebeat again, indexing starts from the beginning again.\n" + - " registry_file: /var/lib/filebeat/registry\n" + - "\n" + - " # Full Path to directory with additional prospector configuration files. Each file must end with .yml\n" + - " # These config files must have the full filebeat config part inside, but only\n" + - " # the prospector part is processed. All global options like spool_size are ignored.\n" + - " # The config_dir MUST point to a different directory then where the main filebeat config file is in.\n" + - " #config_dir:\n" + - "\n" + - "###############################################################################\n" + - "############################# Libbeat Config ##################################\n" + - "# Base config file used by all other beats for using libbeat features\n" + - "\n" + - "############################# Output ##########################################\n" + - "\n" + - "# Configure what outputs to use when sending the data collected by the beat.\n" + - "# Multiple outputs may be used.\n" + - "output:\n" + - "\n" + - " ### Logstash as output\n" + - " logstash:\n" + - " # The Logstash hosts\n" + - " hosts: [%%LOGSTASH_HOSTS%%]\n" + - "\n" + - " timeout: 15\n" + - "\n" + - " # Number of workers per Logstash host.\n" + - " worker: %%LOGSTASH_WORKERS%%\n" + - "\n" + - " # Set gzip compression level.\n" + - " compression_level: 3\n" + - "\n" + - " # Optional load balance the events between the Logstash hosts\n" + - " loadbalance: true\n" + - "\n" + - " # Optional index name. The default index name depends on the each beat.\n" + - " # For Packetbeat, the default is set to packetbeat, for Topbeat\n" + - " # top topbeat and for Filebeat to filebeat.\n" + - " #index: filebeat\n" + - "\n" + - " bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%%\n" + - "\n" + - " # Optional TLS. By default is off.\n" + - " #tls:\n" + - " # List of root certificates for HTTPS server verifications\n" + - " #certificate_authorities: [\"/etc/pki/root/ca.pem\"]\n" + - "\n" + - " # Certificate for TLS client authentication\n" + - " #certificate: \"/etc/pki/client/cert.pem\"\n" + - "\n" + - " # Client Certificate Key\n" + - " #certificate_key: \"/etc/pki/client/cert.key\"\n" + - "\n" + - " # Controls whether the client verifies server certificates and host name.\n" + - " # If insecure is set to true, all server host names and certificates will be\n" + - " # accepted. In this mode TLS based connections are susceptible to\n" + - " # man-in-the-middle attacks. Use only for testing.\n" + - " #insecure: true\n" + - "\n" + - " # Configure cipher suites to be used for TLS connections\n" + - " #cipher_suites: []\n" + - "\n" + - " # Configure curve types for ECDHE based cipher suites\n" + - " #curve_types: []\n" + - "\n" + - "############################# Shipper #########################################\n" + - "\n" + - "shipper:\n" + - "\n" + - "############################# Logging #########################################\n" + - "\n" + - "# There are three options for the log ouput: syslog, file, stderr.\n" + - "# Under Windos systems, the log files are per default sent to the file output,\n" + - "# under all other system per default to syslog.\n" + - "logging:\n" + - "\n" + - " # Send all logging output to syslog. On Windows default is false, otherwise\n" + - " # default is true.\n" + - " to_syslog: false\n" + - "\n" + - " # Write all logging output to files. Beats automatically rotate files if rotateeverybytes\n" + - " # limit is reached.\n" + - " to_files: true\n" + - "\n" + - " # To enable logging to files, to_files option has to be set to true\n" + - " files:\n" + - " # The directory where the log files will written to.\n" + - " path: " + context.pathInNodeUnderVespaHome("logs/filebeat") + "\n" + - "\n" + - " # The name of the files where the logs are written to.\n" + - " name: filebeat\n" + - "\n" + - " # Configure log file size limit. If limit is reached, log file will be\n" + - " # automatically rotated\n" + - " rotateeverybytes: 10485760 # = 10MB\n" + - "\n" + - " # Number of rotated log files to keep. Oldest files will be deleted first.\n" + - " keepfiles: 7\n" + - "\n" + - " # Enable debug output for selected components. To enable all selectors use [\"*\"]\n" + - " # Other available selectors are beat, publish, service\n" + - " # Multiple selectors can be chained.\n" + - " #selectors: [ ]\n" + - "\n" + - " # Sets log level. The default log level is error.\n" + - " # Available log levels are: critical, error, warning, info, debug\n" + - " level: warning\n"; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java deleted file mode 100644 index cf010121c2a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author freva - */ -public class FileHelper { - private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName()); - - /** - * (Recursively) deletes files if they match all the criteria, also deletes empty directories. - * - * @param basePath Base path from where to start the search - * @param maxAge Delete files older (last modified date) than maxAge - * @param fileNameRegex Delete files where filename matches fileNameRegex - * @param recursive Delete files in sub-directories (with the same criteria) - */ - public static void deleteFiles(Path basePath, Duration maxAge, Optional<String> fileNameRegex, boolean recursive) throws IOException { - Pattern fileNamePattern = fileNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - if (recursive) { - deleteFiles(path, maxAge, fileNameRegex, true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } else if (isPatternMatchingFilename(fileNamePattern, path) && - isTimeSinceLastModifiedMoreThan(path, maxAge)) { - if (! Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes all files in target directory except the n most recent (by modified date) - * - * @param basePath Base path to delete from - * @param nMostRecentToKeep Number of most recent files to keep - */ - static void deleteFilesExceptNMostRecent(Path basePath, int nMostRecentToKeep) throws IOException { - if (nMostRecentToKeep < 1) { - throw new IllegalArgumentException("Number of files to keep must be a positive number"); - } - - List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream() - .filter(Files::isRegularFile) - .sorted(Comparator.comparing(FileHelper::getLastModifiedTime)) - .skip(nMostRecentToKeep) - .collect(Collectors.toList()); - - for (Path path : pathsInDeleteDir) { - if (!Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - - static void deleteFilesLargerThan(Path basePath, long sizeInBytes) throws IOException { - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - deleteFilesLargerThan(path, sizeInBytes); - } else { - if (Files.size(path) > sizeInBytes && !Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes directories and their contents if they match all the criteria - * - * @param basePath Base path to delete the directories from - * @param maxAge Delete directories older (last modified date) than maxAge - * @param dirNameRegex Delete directories where directory name matches dirNameRegex - */ - public static void deleteDirectories(Path basePath, Duration maxAge, Optional<String> dirNameRegex) throws IOException { - Pattern dirNamePattern = dirNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path) && isPatternMatchingFilename(dirNamePattern, path)) { - boolean mostRecentFileModifiedBeforeMaxAge = getMostRecentlyModifiedFileIn(path) - .map(mostRecentlyModified -> isTimeSinceLastModifiedMoreThan(mostRecentlyModified, maxAge)) - .orElse(true); - - if (mostRecentFileModifiedBeforeMaxAge) { - deleteFiles(path, Duration.ZERO, Optional.empty(), true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } - } - } - - /** - * Similar to rm -rf file: - * - It's not an error if file doesn't exist - * - If file is a directory, it and all content is removed - * - For symlinks: Only the symlink is removed, not what the symlink points to - */ - public static void recursiveDelete(Path basePath) throws IOException { - if (Files.isDirectory(basePath)) { - for (Path path : listContentsOfDirectory(basePath)) { - recursiveDelete(path); - } - } - - Files.deleteIfExists(basePath); - } - - public static void moveIfExists(Path from, Path to) throws IOException { - if (Files.exists(from)) { - Files.move(from, to); - } - } - - private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException { - return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime)); - } - - private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) { - Instant nowMinusDuration = Instant.now().minus(duration); - Instant lastModified = getLastModifiedTime(path).toInstant(); - - // Return true also if they are equal for test stability - // (lastModified <= nowMinusDuration) is the same as !(lastModified > nowMinusDuration) - return !lastModified.isAfter(nowMinusDuration); - } - - private static boolean isPatternMatchingFilename(Pattern pattern, Path path) { - return pattern == null || pattern.matcher(path.getFileName().toString()).find(); - } - - /** - * @return list all files in a directory, returns empty list if directory does not exist - */ - public static List<Path> listContentsOfDirectory(Path basePath) { - try (Stream<Path> directoryStream = Files.list(basePath)) { - return directoryStream.collect(Collectors.toList()); - } catch (NoSuchFileException ignored) { - return Collections.emptyList(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e); - } - } - - static FileTime getLastModifiedTime(Path path) { - try { - return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS); - } catch (IOException e) { - throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index c9e4a8fac7d..390e81affb2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -24,6 +24,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -55,71 +56,81 @@ public class StorageMaintainer { public void writeMetricsConfig(NodeAgentContext context, NodeSpec node) { List<SecretAgentCheckConfig> configs = new ArrayList<>(); + Map<String, Object> tags = generateTags(context, node); // host-life Path hostLifeCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_host_life"); - SecretAgentCheckConfig hostLifeSchedule = new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath); - configs.add(annotatedCheck(context, node, hostLifeSchedule)); + configs.add(new SecretAgentCheckConfig("host-life", 60, hostLifeCheckPath).withTags(tags)); // ntp Path ntpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ntp"); - SecretAgentCheckConfig ntpSchedule = new SecretAgentCheckConfig("ntp", 60, ntpCheckPath); - configs.add(annotatedCheck(context, node, ntpSchedule)); + configs.add(new SecretAgentCheckConfig("ntp", 60, ntpCheckPath).withTags(tags)); // coredumps (except for the done coredumps which is handled by the host) Path coredumpCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_coredumps"); - SecretAgentCheckConfig coredumpSchedule = new SecretAgentCheckConfig("system-coredumps-processing", 300, - coredumpCheckPath, "--application", "system-coredumps-processing", "--lastmin", - "129600", "--crit", "1", "--coredir", context.pathInNodeUnderVespaHome("var/crash/processing").toString()); - configs.add(annotatedCheck(context, node, coredumpSchedule)); + configs.add(new SecretAgentCheckConfig("system-coredumps-processing", 300, coredumpCheckPath, + "--application", "system-coredumps-processing", + "--lastmin", "129600", + "--crit", "1", + "--coredir", context.pathInNodeUnderVespaHome("var/crash/processing").toString()) + .withTags(tags)); // athenz certificate check Path athenzCertExpiryCheckPath = context.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs"); - SecretAgentCheckConfig athenzCertExpirySchedule = new SecretAgentCheckConfig("athenz-certificate-expiry", 60, - athenzCertExpiryCheckPath, "--threshold", "20") - .withRunAsUser("root"); - configs.add(annotatedCheck(context, node, athenzCertExpirySchedule)); + configs.add(new SecretAgentCheckConfig("athenz-certificate-expiry", 60, athenzCertExpiryCheckPath, + "--threshold", "20") + .withRunAsUser("root") + .withTags(tags)); if (context.nodeType() != NodeType.config) { // vespa-health Path vespaHealthCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health"); - SecretAgentCheckConfig vespaHealthSchedule = new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all"); - configs.add(annotatedCheck(context, node, vespaHealthSchedule)); + configs.add(new SecretAgentCheckConfig("vespa-health", 60, vespaHealthCheckPath, "all").withTags(tags)); // vespa Path vespaCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa"); SecretAgentCheckConfig vespaSchedule = new SecretAgentCheckConfig("vespa", 60, vespaCheckPath, "all"); - configs.add(annotatedCheck(context, node, vespaSchedule)); + if (isConfigserverLike(context.nodeType())) { + Map<String, Object> tagsWithoutNameSpace = new LinkedHashMap<>(tags); + tagsWithoutNameSpace.remove("namespace"); + vespaSchedule.withTags(tagsWithoutNameSpace); + } + configs.add(vespaSchedule); } - if (context.nodeType() == NodeType.config) { + if (context.nodeType() == NodeType.config || context.nodeType() == NodeType.controller) { // configserver Path configServerCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ymonsb2"); - SecretAgentCheckConfig configServerSchedule = new SecretAgentCheckConfig("configserver", 60, - configServerCheckPath, "-zero", "configserver"); - configs.add(annotatedCheck(context, node, configServerSchedule)); + configs.add(new SecretAgentCheckConfig(SecretAgentCheckConfig.nodeTypeToRole(context.nodeType()), 60, configServerCheckPath, + "-zero", "configserver") + .withTags(tags)); //zkbackupage Path zkbackupCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); - SecretAgentCheckConfig zkbackupSchedule = new SecretAgentCheckConfig("zkbackupage", 300, - zkbackupCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(), - "-m", "150", "-a", "config-zkbackupage"); - configs.add(annotatedCheck(context, node, zkbackupSchedule)); + configs.add(new SecretAgentCheckConfig("zkbackupage", 300, zkbackupCheckPath, + "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/zkbackup.stat").toString(), + "-m", "150", + "-a", "config-zkbackupage") + .withTags(tags)); } if (context.nodeType() == NodeType.proxy) { //routing-configage Path routingAgeCheckPath = context.pathInNodeUnderVespaHome("libexec/yamas2/yms_check_file_age.py"); - SecretAgentCheckConfig routingAgeSchedule = new SecretAgentCheckConfig("routing-configage", 60, - routingAgeCheckPath, "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf.tmp").toString(), - "-m", "1", "-a", "routing-configage", "--ignore_file_not_found"); - configs.add(annotatedCheck(context, node, routingAgeSchedule)); + configs.add(new SecretAgentCheckConfig("routing-configage", 60, routingAgeCheckPath, + "-f", context.pathInNodeUnderVespaHome("var/vespa-hosted/routing/nginx.conf.tmp").toString(), + "-m", "1", + "-a", "routing-configage", + "--ignore_file_not_found") + .withTags(tags)); //ssl-check Path sslCheckPath = context.pathInNodeUnderVespaHome("libexec/yms/yms_check_ssl_status"); - SecretAgentCheckConfig sslSchedule = new SecretAgentCheckConfig("ssl-status", 300, - sslCheckPath, "-e", "localhost", "-p", "4443", "-t", "30"); - configs.add(annotatedCheck(context, node, sslSchedule)); + configs.add(new SecretAgentCheckConfig("ssl-status", 300, sslCheckPath, + "-e", "localhost", + "-p", "4443", + "-t", "30") + .withTags(tags)); } // Write config and restart yamas-agent @@ -128,26 +139,36 @@ public class StorageMaintainer { dockerOperations.executeCommandInContainerAsRoot(context, "service", "yamas-agent", "restart"); } - private SecretAgentCheckConfig annotatedCheck(NodeAgentContext context, NodeSpec node, SecretAgentCheckConfig check) { - check.withTag("namespace", "Vespa") - .withTag("role", SecretAgentCheckConfig.nodeTypeToRole(node.getNodeType())) - .withTag("flavor", node.getFlavor()) - .withTag("canonicalFlavor", node.getCanonicalFlavor()) - .withTag("state", node.getState().toString()) - .withTag("zone", String.format("%s.%s", context.zoneId().environment().value(), context.zoneId().regionName().value())); - node.getParentHostname().ifPresent(parent -> check.withTag("parentHostname", parent)); - node.getOwner().ifPresent(owner -> check - .withTag("tenantName", owner.getTenant()) - .withTag("app", owner.getApplication() + "." + owner.getInstance()) - .withTag("applicationName", owner.getApplication()) - .withTag("instanceName", owner.getInstance()) - .withTag("applicationId", owner.getTenant() + "." + owner.getApplication() + "." + owner.getInstance())); - node.getMembership().ifPresent(membership -> check - .withTag("clustertype", membership.getClusterType()) - .withTag("clusterid", membership.getClusterId())); - node.getVespaVersion().ifPresent(version -> check.withTag("vespaVersion", version)); - - return check; + private Map<String, Object> generateTags(NodeAgentContext context, NodeSpec node) { + Map<String, String> tags = new LinkedHashMap<>(); + tags.put("namespace", "Vespa"); + tags.put("role", SecretAgentCheckConfig.nodeTypeToRole(node.getNodeType())); + tags.put("zone", String.format("%s.%s", context.zoneId().environment().value(), context.zoneId().regionName().value())); + node.getVespaVersion().ifPresent(version -> tags.put("vespaVersion", version)); + + if (! isConfigserverLike(context.nodeType())) { + tags.put("flavor", node.getFlavor()); + tags.put("canonicalFlavor", node.getCanonicalFlavor()); + tags.put("state", node.getState().toString()); + node.getParentHostname().ifPresent(parent -> tags.put("parentHostname", parent)); + node.getOwner().ifPresent(owner -> { + tags.put("tenantName", owner.getTenant()); + tags.put("app", owner.getApplication() + "." + owner.getInstance()); + tags.put("applicationName", owner.getApplication()); + tags.put("instanceName", owner.getInstance()); + tags.put("applicationId", owner.getTenant() + "." + owner.getApplication() + "." + owner.getInstance()); + }); + node.getMembership().ifPresent(membership -> { + tags.put("clustertype", membership.getClusterType()); + tags.put("clusterid", membership.getClusterId()); + }); + } + + return Collections.unmodifiableMap(tags); + } + + private boolean isConfigserverLike(NodeType nodeType) { + return nodeType == NodeType.config || nodeType == NodeType.controller; } public Optional<Long> getDiskUsageFor(NodeAgentContext context) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index bbea505b19d..22093258930 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -70,7 +70,7 @@ public class AthenzCredentialsMaintainer { private final CsrGenerator csrGenerator; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts - private Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); + private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); public AthenzCredentialsMaintainer(URI ztsEndpoint, Path trustStorePath, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 395b4d458a2..a220557ca9c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -236,26 +237,33 @@ public class NodeAgentImpl implements NodeAgent { } private void updateNodeRepoWithCurrentAttributes(final NodeSpec node) { - final NodeAttributes currentNodeAttributes = new NodeAttributes() - .withRestartGeneration(node.getCurrentRestartGeneration()) - .withRebootGeneration(node.getCurrentRebootGeneration()) - .withDockerImage(node.getCurrentDockerImage().orElse(new DockerImage(""))); - - final NodeAttributes wantedNodeAttributes = new NodeAttributes() - .withRestartGeneration(node.getWantedRestartGeneration()) - // update reboot gen with wanted gen if set, we ignore reboot for Docker nodes but - // want the two to be equal in node repo - .withRebootGeneration(node.getWantedRebootGeneration()) - .withDockerImage(node.getWantedDockerImage().filter(n -> containerState == UNKNOWN).orElse(new DockerImage(""))); - - publishStateToNodeRepoIfChanged(currentNodeAttributes, wantedNodeAttributes); + final NodeAttributes currentNodeAttributes = new NodeAttributes(); + final NodeAttributes newNodeAttributes = new NodeAttributes(); + + if (!Objects.equals(node.getCurrentRestartGeneration(), node.getWantedRestartGeneration())) { + currentNodeAttributes.withRestartGeneration(node.getCurrentRestartGeneration()); + newNodeAttributes.withRestartGeneration(node.getWantedRestartGeneration()); + } + + if (!Objects.equals(node.getCurrentRebootGeneration(), node.getWantedRebootGeneration())) { + currentNodeAttributes.withRebootGeneration(node.getCurrentRebootGeneration()); + newNodeAttributes.withRebootGeneration(node.getWantedRebootGeneration()); + } + + Optional<DockerImage> actualDockerImage = node.getWantedDockerImage().filter(n -> containerState == UNKNOWN); + if (!Objects.equals(node.getCurrentDockerImage(), actualDockerImage)) { + currentNodeAttributes.withDockerImage(node.getCurrentDockerImage().orElse(new DockerImage(""))); + newNodeAttributes.withDockerImage(actualDockerImage.orElse(new DockerImage(""))); + } + + publishStateToNodeRepoIfChanged(currentNodeAttributes, newNodeAttributes); } - private void publishStateToNodeRepoIfChanged(NodeAttributes currentAttributes, NodeAttributes wantedAttributes) { - if (!currentAttributes.equals(wantedAttributes)) { + private void publishStateToNodeRepoIfChanged(NodeAttributes currentAttributes, NodeAttributes newAttributes) { + if (!currentAttributes.equals(newAttributes)) { context.log(logger, "Publishing new set of attributes to node repo: %s -> %s", - currentAttributes, wantedAttributes); - nodeRepository.updateNodeAttributes(context.hostname().value(), wantedAttributes); + currentAttributes, newAttributes); + nodeRepository.updateNodeAttributes(context.hostname().value(), newAttributes); } } @@ -285,8 +293,8 @@ public class NodeAgentImpl implements NodeAgent { private Optional<String> shouldRestartServices(NodeSpec node) { if (!node.getWantedRestartGeneration().isPresent()) return Optional.empty(); - if (!node.getCurrentRestartGeneration().isPresent() || - node.getCurrentRestartGeneration().get() < node.getWantedRestartGeneration().get()) { + // Restart generation is only optional because it does not exist for unallocated nodes + if (node.getCurrentRestartGeneration().get() < node.getWantedRestartGeneration().get()) { return Optional.of("Restart requested - wanted restart generation has been bumped: " + node.getCurrentRestartGeneration().get() + " -> " + node.getWantedRestartGeneration().get()); } @@ -337,7 +345,7 @@ public class NodeAgentImpl implements NodeAgent { } if (node.getWantedDockerImage().isPresent() && !node.getWantedDockerImage().get().equals(existingContainer.image)) { return Optional.of("The node is supposed to run a new Docker image: " - + existingContainer + " -> " + node.getWantedDockerImage().get()); + + existingContainer.image.asString() + " -> " + node.getWantedDockerImage().get().asString()); } if (!existingContainer.state.isRunning()) { return Optional.of("Container no longer running"); @@ -350,6 +358,11 @@ public class NodeAgentImpl implements NodeAgent { wantedContainerResources + ", actual: " + existingContainer.resources); } + if (node.getCurrentRebootGeneration() < node.getWantedRebootGeneration()) { + return Optional.of(String.format("Container reboot wanted. Current: %d, Wanted: %d", + node.getCurrentRebootGeneration(), node.getWantedRebootGeneration())); + } + if (containerState == STARTING) return Optional.of("Container failed to start"); return Optional.empty(); } @@ -466,7 +479,6 @@ public class NodeAgentImpl implements NodeAgent { new IllegalStateException(String.format("Node '%s' missing from node repository", context.hostname()))); expectNodeNotInNodeRepo = false; - Optional<Container> container = getContainer(); if (!node.equals(lastNode)) { // Every time the node spec changes, we should clear the metrics for this container as the dimensions diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java index 6e679af4449..cdf67871a1a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfig.java @@ -40,6 +40,12 @@ public class SecretAgentCheckConfig { return this; } + public SecretAgentCheckConfig withTags(Map<String, Object> tags) { + this.tags.clear(); + this.tags.putAll(tags); + return this; + } + public void setTags(Map<String, Object> tags) { this.tags.clear(); this.tags.putAll(tags); @@ -63,14 +69,17 @@ public class SecretAgentCheckConfig { .append(" check: ").append(checkExecutable.toFile()).append("\n"); if (arguments.length > 0) { - stringBuilder.append(" args: \n"); + stringBuilder.append(" args:\n"); for (String arg : arguments) { stringBuilder.append(" - ").append(arg).append("\n"); } } - if (!tags.isEmpty()) stringBuilder.append(" tags:\n"); - tags.forEach((key, value) -> stringBuilder.append(" ").append(key).append(": ").append(value).append("\n")); + if (!tags.isEmpty()) { + stringBuilder.append(" tags:\n"); + tags.forEach((key, value) -> + stringBuilder.append(" ").append(key).append(": ").append(value).append("\n")); + } return stringBuilder.toString(); } diff --git a/node-admin/src/main/resources/configdefinitions/config-server.def b/node-admin/src/main/resources/configdefinitions/config-server.def deleted file mode 100644 index 6a088829bad..00000000000 --- a/node-admin/src/main/resources/configdefinitions/config-server.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.hosted.node.admin.config - -hosts[] string -port int default=8080 range=[1,65535] -scheme string default="http" -loadBalancerHost string default="" -configserverAthenzIdentity string default="vespa.configserver"
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java deleted file mode 100644 index 281c7df9a1f..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.component; - -import org.junit.Test; - -import java.nio.file.Paths; - -import static org.junit.Assert.assertEquals; - -public class PathResolverTest { - @Test - public void testNodeAdminOnHost() { - PathResolver pathResolver = new PathResolver(Paths.get("/"), Paths.get("/home/y")); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForHost()); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForNodeAdmin()); - assertEquals(Paths.get("/"), pathResolver.getPathToRootOfHost()); - assertEquals(Paths.get("/home/y"), pathResolver.getVespaHomePathForContainer()); - } - - @Test - public void testNodeAdminInContainer() { - PathResolver pathResolver = new PathResolver(Paths.get("/host"), Paths.get("/home/y")); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForHost()); - assertEquals(Paths.get("/host/home/docker/container-storage"), pathResolver.getApplicationStoragePathForNodeAdmin()); - assertEquals(Paths.get("/host"), pathResolver.getPathToRootOfHost()); - assertEquals(Paths.get("/home/y"), pathResolver.getVespaHomePathForContainer()); - } -}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java index da7b1e6a00c..2cff62afad6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java @@ -26,7 +26,7 @@ import static org.hamcrest.junit.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java index 01aaa385d85..2604aa05367 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/state/StateImplTest.java @@ -8,7 +8,7 @@ import org.junit.Test; import java.net.ConnectException; import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 2a4bf6bf488..6e8cfce6c37 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -28,9 +28,8 @@ import java.util.OptionalLong; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; import static org.mockito.AdditionalMatchers.aryEq; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyVararg; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -50,7 +49,7 @@ public class DockerOperationsImplTest { final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); final ProcessResult actualResult = new ProcessResult(0, "output", "errors"); - when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg())) + when(docker.executeInContainerAsUser(any(), any(), any(), any())) .thenReturn(actualResult); // output from node program ProcessResult result = dockerOperations.executeNodeCtlInContainer(context, "start"); @@ -71,7 +70,7 @@ public class DockerOperationsImplTest { final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); final ProcessResult actualResult = new ProcessResult(3, "output", "errors"); - when(docker.executeInContainerAsUser(any(), any(), any(), anyVararg())) + when(docker.executeInContainerAsUser(any(), any(), any(), any())) .thenReturn(actualResult); // output from node program dockerOperations.executeNodeCtlInContainer(context, "start"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java deleted file mode 100644 index df351d4cc2e..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/CallOrderVerifier.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integrationTests; - -import java.util.Iterator; -import java.util.LinkedList; - -import static org.junit.Assert.assertTrue; - -/** - * Takes in strings representing function calls with their parameters and allows to check whether a subset of calls - * occurred in a specific order. For example, by calling {@link CallOrderVerifier#add(String)} - * with A, B, C, D, E, D, A, F, D and G, then {@link CallOrderVerifier#verifyInOrder(String...)} with - * A, B, D => true, - * A, D, A => true, - * C, D, F => true, - * B, D, A => true - * B, F, D, A => false, - * C, B => false - * - * @author freva - */ -public class CallOrderVerifier { - private static final int waitForCallOrderTimeout = 600; //ms - - private final LinkedList<String> callOrder = new LinkedList<>(); - private final Object monitor = new Object(); - - public void add(String call) { - synchronized (monitor) { - callOrder.add(call); - } - } - - public void assertInOrder(String... functionCalls) { - assertInOrder(waitForCallOrderTimeout, functionCalls); - } - - public void assertInOrder(long timeout, String... functionCalls) { - assertInOrderWithAssertMessage(timeout, "", functionCalls); - } - - public void assertInOrderWithAssertMessage(String assertMessage, String... functionCalls) { - assertInOrderWithAssertMessage(waitForCallOrderTimeout, assertMessage, functionCalls); - } - - public void assertInOrderWithAssertMessage(long timeout, String assertMessage, String... functionCalls) { - boolean inOrder = verifyInOrder(timeout, functionCalls); - if ( ! inOrder && ! assertMessage.isEmpty()) - System.err.println(assertMessage); - assertTrue(toString(), inOrder); - } - - /** - * Checks if list of function calls occur in order given within a timeout - * @param timeout Max number of milliseconds to check for if function calls occur in order - * @param functionCalls The expected order of function calls - * @return true if the actual order of calls was equal to the order provided within timeout, false otherwise. - */ - private boolean verifyInOrder(long timeout, String... functionCalls) { - final long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < timeout) { - if (verifyInOrder(functionCalls)) { - return true; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - return false; - } - - private boolean verifyInOrder(String... functionCalls) { - int pos = 0; - synchronized (monitor) { - for (String functionCall : functionCalls) { - int temp = indexOf(callOrder.listIterator(pos), functionCall); - if (temp == -1) { - System.out.println("Function call '" + functionCall + - "' never made at position " + pos); - return false; - } - pos += temp; - } - } - - return true; - } - - /** - * Finds the first index of needle in haystack after a given position. - * @param iter Iterator to search in - * @param search Element to find in iterator - * @return Index of the next search in after startPos, -1 if not found - */ - private int indexOf(Iterator<String> iter, String search) { - for (int i = 0; iter.hasNext(); i++) { - if (search.equals(iter.next())) { - return i; - } - } - - return -1; - } - - @Override - public String toString() { - synchronized (monitor) { - return callOrder.toString(); - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java index d7e0fe3d8e5..b3bf2282988 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java @@ -3,22 +3,32 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + /** * @author freva */ public class DockerFailTest { @Test - public void dockerFailTest() throws Exception { - try (DockerTester dockerTester = new DockerTester()) { - NodeSpec nodeSpec = new NodeSpec.Builder() - .hostname("host1.test.yahoo.com") - .wantedDockerImage(new DockerImage("dockerImage")) + public void dockerFailTest() { + try (DockerTester tester = new DockerTester()) { + final DockerImage dockerImage = new DockerImage("dockerImage"); + final ContainerName containerName = new ContainerName("host1"); + final String hostname = "host1.test.yahoo.com"; + tester.addChildNodeRepositoryNode(new NodeSpec.Builder() + .hostname(hostname) + .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) .state(Node.State.active) .nodeType(NodeType.tenant) .flavor("docker") @@ -27,24 +37,22 @@ public class DockerFailTest { .minCpuCores(1) .minMainMemoryAvailableGb(1) .minDiskAvailableGb(1) - .build(); - dockerTester.addChildNodeRepositoryNode(nodeSpec); + .build()); - // Wait for node admin to be notified with node repo state and the docker container has been started - while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) { - Thread.sleep(10); - } + tester.inOrder(tester.docker).createContainerCommand( + eq(dockerImage), eq(ContainerResources.from(1, 1)), eq(containerName), eq(hostname)); + tester.inOrder(tester.docker).executeInContainerAsUser( + eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume")); - dockerTester.callOrderVerifier.assertInOrder(1200, - "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + tester.docker.deleteContainer(new ContainerName("host1")); - dockerTester.dockerMock.deleteContainer(new ContainerName("host1")); + tester.inOrder(tester.docker).deleteContainer(eq(containerName)); + tester.inOrder(tester.docker).createContainerCommand( + eq(dockerImage), eq(ContainerResources.from(1, 1)), eq(containerName), eq(hostname)); + tester.inOrder(tester.docker).executeInContainerAsUser( + eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume")); - dockerTester.callOrderVerifier.assertInOrder( - "deleteContainer with ContainerName { name=host1 }", - "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + verify(tester.nodeRepository, never()).updateNodeAttributes(any(), any()); } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java index 26843b119de..75bdaed60cf 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java @@ -13,7 +13,6 @@ import java.net.InetAddress; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,13 +26,8 @@ import java.util.OptionalLong; */ public class DockerMock implements Docker { private final Map<ContainerName, Container> containersByContainerName = new HashMap<>(); - public final CallOrderVerifier callOrderVerifier; private static final Object monitor = new Object(); - public DockerMock(CallOrderVerifier callOrderVerifier) { - this.callOrderVerifier = callOrderVerifier; - } - @Override public CreateContainerCommand createContainerCommand( DockerImage dockerImage, @@ -41,8 +35,6 @@ public class DockerMock implements Docker { ContainerName containerName, String hostName) { synchronized (monitor) { - callOrderVerifier.add("createContainerCommand with " + dockerImage + - ", HostName: " + hostName + ", " + containerName); containersByContainerName.put( containerName, new Container(hostName, dockerImage, containerResources, containerName, Container.State.RUNNING, 2)); } @@ -64,15 +56,12 @@ public class DockerMock implements Docker { @Override public void startContainer(ContainerName containerName) { - synchronized (monitor) { - callOrderVerifier.add("startContainer with " + containerName); - } + } @Override public void stopContainer(ContainerName containerName) { synchronized (monitor) { - callOrderVerifier.add("stopContainer with " + containerName); Container container = containersByContainerName.get(containerName); containersByContainerName.put(containerName, new Container(container.hostname, container.image, container.resources, container.name, Container.State.EXITED, 0)); @@ -82,7 +71,6 @@ public class DockerMock implements Docker { @Override public void deleteContainer(ContainerName containerName) { synchronized (monitor) { - callOrderVerifier.add("deleteContainer with " + containerName); containersByContainerName.remove(containerName); } } @@ -97,7 +85,6 @@ public class DockerMock implements Docker { @Override public boolean pullImageAsyncIfNeeded(DockerImage image) { synchronized (monitor) { - callOrderVerifier.add("pullImageAsyncIfNeeded with " + image); return false; } } @@ -109,9 +96,6 @@ public class DockerMock implements Docker { @Override public ProcessResult executeInContainerAsUser(ContainerName containerName, String user, OptionalLong timeout, String... args) { - synchronized (monitor) { - callOrderVerifier.add("executeInContainer " + containerName.asString() + " as " + user + ", args: " + Arrays.toString(args)); - } return new ProcessResult(0, null, ""); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index aa40d5d805b..6ed51657d6e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -9,8 +9,10 @@ import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; @@ -19,6 +21,8 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesMock; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.test.file.TestFileSystem; +import org.mockito.InOrder; +import org.mockito.Mockito; import java.nio.file.FileSystem; import java.nio.file.Path; @@ -30,8 +34,10 @@ import java.util.Optional; import java.util.function.Function; import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; -import static org.mockito.Matchers.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.when; /** @@ -45,16 +51,19 @@ public class DockerTester implements AutoCloseable { static final String NODE_PROGRAM = PATH_TO_VESPA_HOME.resolve("bin/vespa-nodectl").toString(); static final HostName HOST_HOSTNAME = HostName.from("host.test.yahoo.com"); - final CallOrderVerifier callOrderVerifier = new CallOrderVerifier(); - final Docker dockerMock = new DockerMock(callOrderVerifier); - final NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrderVerifier); + final Docker docker = spy(new DockerMock()); + final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); + final Orchestrator orchestrator = mock(Orchestrator.class); + final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); + final InOrder inOrder = Mockito.inOrder(docker, nodeRepository, orchestrator, storageMaintainer); + final NodeAdminStateUpdaterImpl nodeAdminStateUpdater; final NodeAdminImpl nodeAdmin; - private final OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier); - private final FileSystem fileSystem = TestFileSystem.create(); DockerTester() { + when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.empty()); + IPAddressesMock ipAddresses = new IPAddressesMock(); ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1"); ipAddresses.addAddress(HOST_HOSTNAME.value(), "f000::"); @@ -71,25 +80,25 @@ public class DockerTester implements AutoCloseable { .wantedRestartGeneration(1L) .currentRestartGeneration(1L) .build(); - nodeRepositoryMock.updateNodeRepositoryNode(hostSpec); + nodeRepository.updateNodeRepositoryNode(hostSpec); Clock clock = Clock.systemUTC(); - DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, processExecuter, node -> "", Collections.emptyList(), ipAddresses); - StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerOperations, callOrderVerifier); + FileSystem fileSystem = TestFileSystem.create(); + DockerOperations dockerOperations = new DockerOperationsImpl(docker, processExecuter, node -> "", Collections.emptyList(), ipAddresses); MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl( - new NodeAgentContextImpl.Builder(hostName).fileSystem(fileSystem).build(), nodeRepositoryMock, - orchestratorMock, dockerOperations, storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.empty(), Optional.empty()); + new NodeAgentContextImpl.Builder(hostName).fileSystem(fileSystem).build(), nodeRepository, + orchestrator, dockerOperations, storageMaintainer, clock, NODE_AGENT_SCAN_INTERVAL, Optional.empty(), Optional.empty()); nodeAdmin = new NodeAdminImpl(nodeAgentFactory, Optional.empty(), mr, Clock.systemUTC()); - nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepositoryMock, orchestratorMock, + nodeAdminStateUpdater = new NodeAdminStateUpdaterImpl(nodeRepository, orchestrator, nodeAdmin, HOST_HOSTNAME, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL); nodeAdminStateUpdater.start(); } /** Adds a node to node-repository mock that is running on this host */ void addChildNodeRepositoryNode(NodeSpec nodeSpec) { - nodeRepositoryMock.updateNodeRepositoryNode(new NodeSpec.Builder(nodeSpec) + nodeRepository.updateNodeRepositoryNode(new NodeSpec.Builder(nodeSpec) .parentHostname(HOST_HOSTNAME.value()) .build()); } @@ -98,4 +107,8 @@ public class DockerTester implements AutoCloseable { public void close() { nodeAdminStateUpdater.stop(); } + + public <T> T inOrder(T t) { + return inOrder.verify(t, timeout(1000)); + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index e130cd0d475..48fdc564aac 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -3,24 +3,30 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; + /** * @author freva */ public class MultiDockerTest { @Test - public void test() throws InterruptedException { - try (DockerTester dockerTester = new DockerTester()) { - addAndWaitForNode(dockerTester, "host1.test.yahoo.com", new DockerImage("image1")); + public void test() { + try (DockerTester tester = new DockerTester()) { + addAndWaitForNode(tester, "host1.test.yahoo.com", new DockerImage("image1")); NodeSpec nodeSpec2 = addAndWaitForNode( - dockerTester, "host2.test.yahoo.com", new DockerImage("image2")); + tester, "host2.test.yahoo.com", new DockerImage("image2")); - dockerTester.addChildNodeRepositoryNode( + tester.addChildNodeRepositoryNode( new NodeSpec.Builder(nodeSpec2) .state(Node.State.dirty) .minCpuCores(1) @@ -28,66 +34,37 @@ public class MultiDockerTest { .minDiskAvailableGb(1) .build()); - // Wait until it is marked ready - while (dockerTester.nodeRepositoryMock.getOptionalNode(nodeSpec2.getHostname()) - .filter(node -> node.getState() != Node.State.ready).isPresent()) { - Thread.sleep(10); - } - - addAndWaitForNode(dockerTester, "host3.test.yahoo.com", new DockerImage("image1")); - - dockerTester.callOrderVerifier.assertInOrder( - "createContainerCommand with DockerImage { imageId=image1 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]", - - "createContainerCommand with DockerImage { imageId=image2 }, HostName: host2.test.yahoo.com, ContainerName { name=host2 }", - "executeInContainer host2 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]", - - "stopContainer with ContainerName { name=host2 }", - "deleteContainer with ContainerName { name=host2 }", + tester.inOrder(tester.docker).deleteContainer(eq(new ContainerName("host2"))); + tester.inOrder(tester.storageMaintainer).archiveNodeStorage( + argThat(context -> context.containerName().equals(new ContainerName("host2")))); + tester.inOrder(tester.nodeRepository).setNodeState(eq(nodeSpec2.getHostname()), eq(Node.State.ready)); - "createContainerCommand with DockerImage { imageId=image1 }, HostName: host3.test.yahoo.com, ContainerName { name=host3 }", - "executeInContainer host3 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); - - dockerTester.callOrderVerifier.assertInOrderWithAssertMessage( - "Maintainer did not receive call to delete application storage", - "deleteContainer with ContainerName { name=host2 }", - "DeleteContainerStorage with ContainerName { name=host2 }"); - - dockerTester.callOrderVerifier.assertInOrder( - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}", - "updateNodeAttributes with HostName: host2.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image2}", - "setNodeState host2.test.yahoo.com to ready", - "updateNodeAttributes with HostName: host3.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image1}"); + addAndWaitForNode(tester, "host3.test.yahoo.com", new DockerImage("image1")); } } - private NodeSpec addAndWaitForNode(DockerTester tester, String hostName, DockerImage dockerImage) throws InterruptedException { + private NodeSpec addAndWaitForNode(DockerTester tester, String hostName, DockerImage dockerImage) { NodeSpec nodeSpec = new NodeSpec.Builder() .hostname(hostName) .wantedDockerImage(dockerImage) - .wantedVespaVersion("1.2.3") .state(Node.State.active) .nodeType(NodeType.tenant) .flavor("docker") .wantedRestartGeneration(1L) .currentRestartGeneration(1L) - .minCpuCores(1) - .minMainMemoryAvailableGb(1) + .minCpuCores(2) + .minMainMemoryAvailableGb(4) .minDiskAvailableGb(1) .build(); tester.addChildNodeRepositoryNode(nodeSpec); - // Wait for node admin to be notified with node repo state and the docker container has been started - while (tester.nodeAdmin.getNumberOfNodeAgents() + 1 != tester.nodeRepositoryMock.getNumberOfContainerSpecs()) { - Thread.sleep(10); - } - ContainerName containerName = ContainerName.fromHostname(hostName); - tester.callOrderVerifier.assertInOrder( - "createContainerCommand with " + dockerImage + ", HostName: " + hostName + ", " + containerName, - "executeInContainer " + containerName.asString() + " as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); + tester.inOrder(tester.docker).createContainerCommand( + eq(dockerImage), eq(ContainerResources.from(2, 4)), eq(containerName), eq(hostName)); + tester.inOrder(tester.docker).executeInContainerAsUser( + eq(containerName), eq("root"), any(), eq(DockerTester.NODE_PROGRAM), eq("resume")); + tester.inOrder(tester.nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withDockerImage(dockerImage))); return nodeSpec; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index d25d79ab457..e8b423e73ac 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttribu import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.provision.Node; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,13 +24,6 @@ public class NodeRepoMock implements NodeRepository { private static final Object monitor = new Object(); private final Map<String, NodeSpec> nodeRepositoryNodesByHostname = new HashMap<>(); - private final Map<String, Acl> acls = new HashMap<>(); - - private final CallOrderVerifier callOrderVerifier; - - public NodeRepoMock(CallOrderVerifier callOrderVerifier) { - this.callOrderVerifier = callOrderVerifier; - } @Override public void addNodes(List<AddNode> nodes) { } @@ -52,15 +46,15 @@ public class NodeRepoMock implements NodeRepository { @Override public Map<String, Acl> getAcls(String hostname) { - synchronized (monitor) { - return acls; - } + return Collections.emptyMap(); } @Override public void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes) { synchronized (monitor) { - callOrderVerifier.add("updateNodeAttributes with HostName: " + hostName + ", " + nodeAttributes); + updateNodeRepositoryNode(new NodeSpec.Builder(getNode(hostName)) + .updateFromNodeAttributes(nodeAttributes) + .build()); } } @@ -70,7 +64,6 @@ public class NodeRepoMock implements NodeRepository { updateNodeRepositoryNode(new NodeSpec.Builder(getNode(hostName)) .state(nodeState) .build()); - callOrderVerifier.add("setNodeState " + hostName + " to " + nodeState); } } @@ -79,13 +72,9 @@ public class NodeRepoMock implements NodeRepository { } - public void updateNodeRepositoryNode(NodeSpec nodeSpec) { - nodeRepositoryNodesByHostname.put(nodeSpec.getHostname(), nodeSpec); - } - - public int getNumberOfContainerSpecs() { + void updateNodeRepositoryNode(NodeSpec nodeSpec) { synchronized (monitor) { - return nodeRepositoryNodesByHostname.size(); + nodeRepositoryNodesByHostname.put(nodeSpec.getHostname(), nodeSpec); } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java deleted file mode 100644 index da743c40e8b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integrationTests; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.provision.Node; -import org.junit.Test; - -/** - * Test NodeState transitions in NodeRepository - * - * @author freva - */ -public class NodeStateTest { - private final NodeSpec initialNodeSpec = new NodeSpec.Builder() - .hostname("host1.test.yahoo.com") - .wantedDockerImage(new DockerImage("dockerImage")) - .state(Node.State.active) - .nodeType(NodeType.tenant) - .flavor("docker") - .wantedRestartGeneration(1L) - .currentRestartGeneration(1L) - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - - private void setup(DockerTester tester) throws InterruptedException { - tester.addChildNodeRepositoryNode(initialNodeSpec); - - // Wait for node admin to be notified with node repo state and the docker container has been started - while (tester.nodeAdmin.getNumberOfNodeAgents() == 0) { - Thread.sleep(10); - } - - tester.callOrderVerifier.assertInOrder( - "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); - } - - - @Test - public void activeToDirty() throws InterruptedException { - try (DockerTester dockerTester = new DockerTester()) { - setup(dockerTester); - // Change node state to dirty - dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec) - .state(Node.State.dirty) - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build()); - - // Wait until it is marked ready - while (dockerTester.nodeRepositoryMock.getOptionalNode(initialNodeSpec.getHostname()) - .filter(node -> node.getState() != Node.State.ready).isPresent()) { - Thread.sleep(10); - } - - dockerTester.callOrderVerifier.assertInOrder( - "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", stop]", - "stopContainer with ContainerName { name=host1 }", - "deleteContainer with ContainerName { name=host1 }"); - } - } - - @Test - public void activeToInactiveToActive() throws InterruptedException { - - try (DockerTester dockerTester = new DockerTester()) { - setup(dockerTester); - - DockerImage newDockerImage = new DockerImage("newDockerImage"); - - // Change node state to inactive and change the wanted docker image - dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec) - .wantedDockerImage(newDockerImage) - .state(Node.State.inactive) - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build()); - - dockerTester.callOrderVerifier.assertInOrderWithAssertMessage( - "Node set to inactive, but no stop/delete call received", - "stopContainer with ContainerName { name=host1 }", - "deleteContainer with ContainerName { name=host1 }"); - - - // Change node state to active - dockerTester.addChildNodeRepositoryNode(new NodeSpec.Builder(initialNodeSpec) - .wantedDockerImage(newDockerImage) - .state(Node.State.active) - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build()); - - // Check that the container is started again after the delete call - dockerTester.callOrderVerifier.assertInOrderWithAssertMessage( - "Node not started again after being put to active state", - "deleteContainer with ContainerName { name=host1 }", - "createContainerCommand with DockerImage { imageId=newDockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", resume]"); - } - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java deleted file mode 100644 index 469022cec56..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/OrchestratorMock.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integrationTests; - -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; - -import java.util.List; - -/** - * Mock with some simple logic - * - * @author dybis - */ -public class OrchestratorMock implements Orchestrator { - private final CallOrderVerifier callOrderVerifier; - - OrchestratorMock(CallOrderVerifier callOrderVerifier) { - this.callOrderVerifier = callOrderVerifier; - } - - @Override - public void suspend(String hostName) { - callOrderVerifier.add("Suspend for " + hostName); - } - - @Override - public void resume(String hostName) { - callOrderVerifier.add("Resume for " + hostName); - } - - @Override - public void suspend(String parentHostName, List<String> hostNames) { - callOrderVerifier.add("Suspend with parent: " + parentHostName + " and hostnames: " + hostNames); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java index 778443295bf..75fefde427d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java @@ -2,16 +2,24 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.provider.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl; import com.yahoo.vespa.hosted.provision.Node; -import org.junit.Ignore; import org.junit.Test; +import java.util.Arrays; +import java.util.OptionalLong; + +import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.HOST_HOSTNAME; +import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; /** * Tests rebooting of Docker host @@ -20,47 +28,33 @@ import static org.junit.Assert.assertTrue; */ public class RebootTest { - @Test - @Ignore - public void test() throws InterruptedException { - try (DockerTester dockerTester = new DockerTester()) { - - dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode()); - - // Wait for node admin to be notified with node repo state and the docker container has been started - while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) { - Thread.sleep(10); - } + private final String hostname = "host1.test.yahoo.com"; + private final DockerImage dockerImage = new DockerImage("dockerImage"); - // Check that the container is started and NodeRepo has received the PATCH update - dockerTester.callOrderVerifier.assertInOrder( - "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=null, dockerImage=dockerImage, vespaVersion='null'}"); - - NodeAdminStateUpdaterImpl updater = dockerTester.nodeAdminStateUpdater; -// assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), -// is(Optional.of("Not all node agents are frozen."))); - - updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); + @Test + public void test() { + try (DockerTester tester = new DockerTester()) { + tester.addChildNodeRepositoryNode(createNodeRepositoryNode()); - NodeAdmin nodeAdmin = dockerTester.nodeAdmin; - // Wait for node admin to be frozen - while ( ! nodeAdmin.isFrozen()) { - System.out.println("Node admin not frozen yet"); - Thread.sleep(10); - } + tester.inOrder(tester.docker).createContainerCommand( + eq(dockerImage), eq(ContainerResources.from(0, 0)), eq(new ContainerName("host1")), eq(hostname)); - assertTrue(nodeAdmin.setFrozen(false)); + try { + tester.nodeAdminStateUpdater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); + } catch (RuntimeException ignored) { } - dockerTester.callOrderVerifier.assertInOrder( - "executeInContainer with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", stop]"); + tester.inOrder(tester.orchestrator).suspend( + eq(HOST_HOSTNAME.value()), eq(Arrays.asList(hostname, HOST_HOSTNAME.value()))); + tester.inOrder(tester.docker).executeInContainerAsUser( + eq(new ContainerName("host1")), eq("root"), eq(OptionalLong.empty()), eq(NODE_PROGRAM), eq("stop")); + assertTrue(tester.nodeAdmin.setFrozen(true)); } } private NodeSpec createNodeRepositoryNode() { return new NodeSpec.Builder() - .hostname("host1.test.yahoo.com") - .wantedDockerImage(new DockerImage("dockerImage")) + .hostname(hostname) + .wantedDockerImage(dockerImage) .state(Node.State.active) .nodeType(NodeType.tenant) .flavor("docker") diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index 2e290e014c2..2b8727529f2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -2,11 +2,19 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.DockerImage; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; + /** * Tests that different wanted and current restart generation leads to execution of restart command * @@ -15,46 +23,35 @@ import org.junit.Test; public class RestartTest { @Test - public void test() throws InterruptedException { - try (DockerTester dockerTester = new DockerTester()) { - - long wantedRestartGeneration = 1; - long currentRestartGeneration = wantedRestartGeneration; - dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode(wantedRestartGeneration, currentRestartGeneration)); - - // Wait for node admin to be notified with node repo state and the docker container has been started - while (dockerTester.nodeAdmin.getNumberOfNodeAgents() == 0) { - Thread.sleep(10); - } - - // Check that the container is started and NodeRepo has received the PATCH update - dockerTester.callOrderVerifier.assertInOrder( - "createContainerCommand with DockerImage { imageId=image:1.2.3 }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", - "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=0, dockerImage=image:1.2.3}"); - - wantedRestartGeneration = 2; - currentRestartGeneration = 1; - dockerTester.addChildNodeRepositoryNode(createNodeRepositoryNode(wantedRestartGeneration, currentRestartGeneration)); - - dockerTester.callOrderVerifier.assertInOrder( - "Suspend for host1.test.yahoo.com", - "executeInContainer host1 as root, args: [" + DockerTester.NODE_PROGRAM + ", restart-vespa]"); + public void test() { + try (DockerTester tester = new DockerTester()) { + String hostname = "host1.test.yahoo.com"; + DockerImage dockerImage = new DockerImage("dockerImage:1.2.3"); + + tester.addChildNodeRepositoryNode(new NodeSpec.Builder() + .hostname(hostname) + .state(Node.State.active) + .wantedDockerImage(dockerImage) + .nodeType(NodeType.tenant) + .flavor("docker") + .wantedRestartGeneration(1) + .currentRestartGeneration(1) + .build()); + + tester.inOrder(tester.docker).createContainerCommand( + eq(dockerImage), any(), eq(new ContainerName("host1")), eq(hostname)); + tester.inOrder(tester.nodeRepository).updateNodeAttributes( + eq(hostname), eq(new NodeAttributes().withDockerImage(dockerImage))); + + // Increment wantedRestartGeneration to 2 in node-repo + tester.addChildNodeRepositoryNode(new NodeSpec.Builder(tester.nodeRepository.getNode(hostname)) + .wantedRestartGeneration(2).build()); + + tester.inOrder(tester.orchestrator).suspend(eq(hostname)); + tester.inOrder(tester.docker).executeInContainerAsUser( + eq(new ContainerName("host1")), any(), any(), eq(NODE_PROGRAM), eq("restart-vespa")); + tester.inOrder(tester.nodeRepository).updateNodeAttributes( + eq(hostname), eq(new NodeAttributes().withRestartGeneration(Optional.of(2L)))); } } - - private NodeSpec createNodeRepositoryNode(long wantedRestartGeneration, long currentRestartGeneration) { - return new NodeSpec.Builder() - .hostname("host1.test.yahoo.com") - .state(Node.State.active) - .wantedDockerImage(new DockerImage("image:1.2.3")) - .wantedVespaVersion("1.2.3") - .nodeType(NodeType.tenant) - .flavor("docker") - .wantedRestartGeneration(wantedRestartGeneration) - .currentRestartGeneration(currentRestartGeneration) - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java deleted file mode 100644 index 298c185266b..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/StorageMaintainerMock.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.integrationTests; - -import com.yahoo.vespa.hosted.dockerapi.Container; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; - -import java.util.Optional; - -/** - * @author freva - */ -public class StorageMaintainerMock extends StorageMaintainer { - private final CallOrderVerifier callOrderVerifier; - - public StorageMaintainerMock(DockerOperations dockerOperations, CallOrderVerifier callOrderVerifier) { - super(dockerOperations, null, null); - this.callOrderVerifier = callOrderVerifier; - } - - @Override - public Optional<Long> getDiskUsageFor(NodeAgentContext context) { - return Optional.empty(); - } - - @Override - public void handleCoreDumpsForContainer(NodeAgentContext context, NodeSpec node, Optional<Container> container) { - } - - @Override - public void removeOldFilesFromNode(NodeAgentContext context) { - } - - @Override - public void archiveNodeStorage(NodeAgentContext context) { - callOrderVerifier.add("DeleteContainerStorage with " + context.containerName()); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java deleted file mode 100644 index d4fadabe695..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.logging; - -import com.google.common.collect.ImmutableList; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.component.Environment; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.provision.Node; -import org.junit.Test; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.*; - -/** - * @author mortent - */ -public class FilebeatConfigProviderTest { - - private static final String tenant = "vespa"; - private static final String application = "music"; - private static final String instance = "default"; - private static final String environment = "prod"; - private static final String region = "us-north-1"; - private static final String system = "main"; - private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2"); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("node-123.hostname.tld").build(); - - @Test - public void it_replaces_all_fields_correctly() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - - assertTrue(config.isPresent()); - String configString = config.get(); - assertThat(configString, not(containsString("%%"))); - } - - @Test - public void it_does_not_generate_config_when_no_logstash_nodes() { - Environment env = getEnvironment(Collections.emptyList()); - - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(env); - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - assertFalse(config.isPresent()); - } - - @Test - public void it_does_not_generate_config_for_nodes_wihout_owner() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - NodeSpec node = new NodeSpec.Builder() - .flavor("flavor") - .state(Node.State.active) - .nodeType(NodeType.tenant) - .hostname("hostname") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - Optional<String> config = filebeatConfigProvider.getConfig(context, node); - assertFalse(config.isPresent()); - } - - @Test - public void it_generates_correct_index_source() { - assertThat(getConfigString(), containsString("index_source: \"hosted-instance_vespa_music_us-north-1_prod_default\"")); - } - - @Test - public void it_sets_logstash_nodes_properly() { - assertThat(getConfigString(), containsString("hosts: [\"logstash1\",\"logstash2\"]")); - } - - @Test - public void it_does_not_add_double_quotes() { - Environment environment = getEnvironment(ImmutableList.of("unquoted", "\"quoted\"")); - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment); - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - assertThat(config.get(), containsString("hosts: [\"unquoted\",\"quoted\"]")); - } - - @Test - public void it_generates_correct_spool_size() { - // 2 nodes, 3 workers, 2048 buffer size -> 12288 - assertThat(getConfigString(), containsString("spool_size: 12288")); - } - - private String getConfigString() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - NodeSpec node = createNodeRepositoryNode(tenant, application, instance); - return filebeatConfigProvider.getConfig(context, node).orElseThrow(() -> new RuntimeException("Failed to get filebeat config")); - } - - private Environment getEnvironment(List<String> logstashNodes) { - return new Environment.Builder() - .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) - .environment(environment) - .region(region) - .system(system) - .logstashNodes(logstashNodes) - .cloud("mycloud") - .dockerNetworking(DockerNetworking.HOST_NETWORK) - .build(); - } - - private NodeSpec createNodeRepositoryNode(String tenant, String application, String instance) { - NodeSpec.Owner owner = new NodeSpec.Owner(tenant, application, instance); - return new NodeSpec.Builder() - .owner(owner) - .flavor("flavor") - .state(Node.State.active) - .nodeType(NodeType.tenant) - .hostname("hostname") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java deleted file mode 100644 index 6b53bc217c4..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author freva - */ -public class FileHelperTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void initFiles() throws IOException { - for (int i=0; i<10; i++) { - File temp = folder.newFile("test_" + i + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis()); - } - - for (int i=0; i<7; i++) { - File temp = folder.newFile("test_" + i + "_file.test"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis()); - } - - for (int i=0; i<5; i++) { - File temp = folder.newFile(i + "-abc" + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis()); - } - - File temp = folder.newFile("week_old_file.json"); - temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis()); - } - - @Test - public void testDeleteAll() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.empty(), false); - - assertEquals(0, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(6, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file - } - - @Test - public void testDeleteSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".json$"), false); - - assertEquals(7, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefixAndSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 7 test_*_file.test files + week_old_file - } - - @Test - public void testDeleteOld() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(600), Optional.empty(), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file - } - - @Test - public void testDeleteWithAllParameters() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(200), Optional.of("^test_.*\\.json$"), false); - - assertEquals(15, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 8 (from test_*_.json) - } - - @Test - public void testDeleteWithSubDirectoriesNoRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - // 6 test_*.json from test_folder1/ - // + 9 test_*.json and 4 abc_*.json from test_folder2/ - // + 13 test_*.json from test_folder2/subSubFolder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves - assertEquals(48, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteWithSubDirectoriesRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), true); - - // 4 abc_*.json from test_folder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder2/ itself - assertEquals(18, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(8, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file + 2 directories - } - - @Test - public void testGetContentsOfNonExistingDirectory() { - Path fakePath = Paths.get("/some/made/up/dir/"); - assertEquals(Collections.emptyList(), FileHelper.listContentsOfDirectory(fakePath)); - } - - @Test(expected=IllegalArgumentException.class) - public void testDeleteFilesExceptNMostRecentWithNegativeN() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), -5); - } - - @Test - public void testDeleteFilesExceptFiveMostRecent() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 5); - - assertEquals(5, getContentsOfDirectory(folder.getRoot()).length); - - String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"}; - String[] remainingFiles = Arrays.stream(getContentsOfDirectory(folder.getRoot())) - .map(File::getName) - .sorted() - .toArray(String[]::new); - - assertArrayEquals(oldestFiles, remainingFiles); - } - - @Test - public void testDeleteFilesExceptNMostRecentWithLargeN() throws IOException { - String[] filesPreDelete = folder.getRoot().list(); - - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 50); - - assertArrayEquals(filesPreDelete, folder.getRoot().list()); - } - - @Test - public void testDeleteFilesLargerThan10B() throws IOException { - initSubDirectories(); - - File temp1 = new File(folder.getRoot(), "small_file"); - writeNBytesToFile(temp1, 50); - - File temp2 = new File(folder.getRoot(), "some_file"); - writeNBytesToFile(temp2, 20); - - File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file"); - writeNBytesToFile(temp3, 75); - - FileHelper.deleteFilesLargerThan(folder.getRoot().toPath(), 10); - - assertEquals(58, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - assertFalse(temp1.exists() || temp2.exists() || temp3.exists()); - } - - @Test - public void testDeleteDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".*folder2")); - - //23 files in root - // + 6 in test_folder1 + test_folder1 itself - assertEquals(30, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteDirectoriesBasedOnAge() throws IOException { - initSubDirectories(); - // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is - // also older than maxAge inside the sub directory, create some files which are newer than maxAge. - // deleteDirectories() should NOT delete folder3 - File subFolder3 = folder.newFolder("test_folder3"); - File subSubFolder3 = folder.newFolder("test_folder3", "subSubFolder3"); - - for (int j=0; j<11; j++) { - File.createTempFile("test_", ".json", subSubFolder3); - } - - subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*")); - - //23 files in root - // + 13 in test_folder2 - // + 13 in subSubFolder2 - // + 11 in subSubFolder3 - // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself - assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testRecursivelyDeleteDirectory() throws IOException { - initSubDirectories(); - FileHelper.recursiveDelete(folder.getRoot().toPath()); - assertFalse(folder.getRoot().exists()); - } - - @Test - public void testRecursivelyDeleteRegularFile() throws IOException { - File file = folder.newFile(); - assertTrue(file.exists()); - assertTrue(file.isFile()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testRecursivelyDeleteNonExistingFile() throws IOException { - File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile(); - assertFalse(file.exists()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testInitSubDirectories() throws IOException { - initSubDirectories(); - assertTrue(folder.getRoot().exists()); - assertTrue(folder.getRoot().isDirectory()); - - Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1"); - assertTrue(test_folder1.toFile().exists()); - assertTrue(test_folder1.toFile().isDirectory()); - - Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2"); - assertTrue(test_folder2.toFile().exists()); - assertTrue(test_folder2.toFile().isDirectory()); - - Path subSubFolder2 = test_folder2.resolve("subSubFolder2"); - assertTrue(subSubFolder2.toFile().exists()); - assertTrue(subSubFolder2.toFile().isDirectory()); - } - - @Test - public void testDoesNotFailOnLastModifiedOnSymLink() throws IOException { - Path symPath = folder.getRoot().toPath().resolve("symlink"); - Path fakePath = Paths.get("/some/not/existant/file"); - - Files.createSymbolicLink(symPath, fakePath); - assertTrue(Files.isSymbolicLink(symPath)); - assertFalse(Files.exists(fakePath)); - - // Not possible to set modified time on symlink in java, so just check that it doesn't crash - FileHelper.getLastModifiedTime(symPath).toInstant(); - } - - private void initSubDirectories() throws IOException { - File subFolder1 = folder.newFolder("test_folder1"); - File subFolder2 = folder.newFolder("test_folder2"); - File subSubFolder2 = folder.newFolder("test_folder2", "subSubFolder2"); - - for (int j=0; j<6; j++) { - File temp = File.createTempFile("test_", ".json", subFolder1); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis()); - } - - for (int j=0; j<9; j++) { - File.createTempFile("test_", ".json", subFolder2); - } - - for (int j=0; j<4; j++) { - File.createTempFile("abc_", ".txt", subFolder2); - } - - for (int j=0; j<13; j++) { - File temp = File.createTempFile("test_", ".json", subSubFolder2); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis()); - } - - //Must be after all the files have been created - subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis()); - subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - } - - private static int getNumberOfFilesAndDirectoriesIn(File folder) { - int total = 0; - for (File file : getContentsOfDirectory(folder)) { - if (file.isDirectory()) { - total += getNumberOfFilesAndDirectoriesIn(file); - } - total++; - } - - return total; - } - - private static void writeNBytesToFile(File file, int nBytes) throws IOException { - Files.write(file.toPath(), new byte[nBytes]); - } - - private static File[] getContentsOfDirectory(File directory) { - File[] directoryContents = directory.listFiles(); - - return directoryContents == null ? new File[0] : directoryContents; - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 42b161b8e2d..d537cb40214 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -2,14 +2,24 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.node.admin.component.ZoneId; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Rule; import org.junit.Test; +import org.junit.experimental.runners.Enclosed; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; import java.io.IOException; import java.nio.file.FileSystem; @@ -20,7 +30,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -29,99 +41,247 @@ import static org.mockito.Mockito.mock; /** * @author dybis */ +@RunWith(Enclosed.class) public class StorageMaintainerTest { - private final DockerOperations docker = mock(DockerOperations.class); + private static final DockerOperations docker = mock(DockerOperations.class); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + public static class SecretAgentCheckTests { + private final StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, null); - @Test - public void testDiskUsed() throws IOException, InterruptedException { - StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, null); - int writeSize = 10000; - Files.write(folder.newFile().toPath(), new byte[writeSize]); + @Test + public void tenant() { + Path path = executeAs(NodeType.tenant); - long usedBytes = storageMaintainer.getDiskUsedInBytes(folder.getRoot().toPath()); - if (usedBytes * 4 < writeSize || usedBytes > writeSize * 4) - fail("Used bytes is " + usedBytes + ", but wrote " + writeSize + " bytes, not even close."); - } + assertChecks(path, "athenz-certificate-expiry", "host-life", "ntp", + "system-coredumps-processing", "vespa", "vespa-health"); + + // All dimensions for vespa metrics should be set by metricsproxy + assertCheckEnds(path.resolve("vespa.yaml"), + " args:\n" + + " - all\n"); + + // For non vespa metrics, we need to set all the dimensions ourselves + assertCheckEnds(path.resolve("ntp.yaml"), + "tags:\n" + + " namespace: Vespa\n" + + " role: tenants\n" + + " zone: prod.us-north-1\n" + + " vespaVersion: 6.305.12\n" + + " flavor: d-2-8-50\n" + + " canonicalFlavor: d-2-8-50\n" + + " state: active\n" + + " parentHostname: host123.test.domain.tld\n" + + " tenantName: tenant\n" + + " app: application.instance\n" + + " applicationName: application\n" + + " instanceName: instance\n" + + " applicationId: tenant.application.instance\n" + + " clustertype: clusterType\n" + + " clusterid: clusterId\n"); + } + + @Test + public void proxy() { + Path path = executeAs(NodeType.proxy); + + assertChecks(path, "athenz-certificate-expiry", "host-life", "ntp", "routing-configage", + "ssl-status", "system-coredumps-processing", "vespa", "vespa-health"); + + // All dimensions for vespa metrics should be set by the source + assertCheckEnds(path.resolve("vespa.yaml"), + " args:\n" + + " - all\n"); + + // For non vespa metrics, we need to set all the dimensions ourselves + assertCheckEnds(path.resolve("ntp.yaml"), + "tags:\n" + + " namespace: Vespa\n" + + " role: routing\n" + + " zone: prod.us-north-1\n" + + " vespaVersion: 6.305.12\n" + + " flavor: d-2-8-50\n" + + " canonicalFlavor: d-2-8-50\n" + + " state: active\n" + + " parentHostname: host123.test.domain.tld\n" + + " tenantName: tenant\n" + + " app: application.instance\n" + + " applicationName: application\n" + + " instanceName: instance\n" + + " applicationId: tenant.application.instance\n" + + " clustertype: clusterType\n" + + " clusterid: clusterId\n"); + } + + @Test + public void configserver() { + Path path = executeAs(NodeType.config); + + assertChecks(path, "athenz-certificate-expiry", "configserver", "host-life", "ntp", + "system-coredumps-processing", "zkbackupage"); + + assertCheckEnds(path.resolve("configserver.yaml"), + " tags:\n" + + " namespace: Vespa\n" + + " role: configserver\n" + + " zone: prod.us-north-1\n" + + " vespaVersion: 6.305.12\n"); + } + + @Test + public void controller() { + Path path = executeAs(NodeType.controller); + + assertChecks(path, "athenz-certificate-expiry", "controller", "host-life", "ntp", + "system-coredumps-processing", "vespa", "vespa-health", "zkbackupage"); + + + // Do not set namespace for vespa metrics. WHY? + assertCheckEnds(path.resolve("vespa.yaml"), + " tags:\n" + + " role: controller\n" + + " zone: prod.us-north-1\n" + + " vespaVersion: 6.305.12\n"); + + assertCheckEnds(path.resolve("controller.yaml"), + " tags:\n" + + " namespace: Vespa\n" + + " role: controller\n" + + " zone: prod.us-north-1\n" + + " vespaVersion: 6.305.12\n"); + } + + private Path executeAs(NodeType nodeType) { + NodeAgentContext context = new NodeAgentContextImpl.Builder("host123-5.test.domain.tld") + .nodeType(nodeType) + .fileSystem(TestFileSystem.create()) + .zoneId(new ZoneId(SystemName.dev, Environment.prod, RegionName.from("us-north-1"))).build(); + NodeSpec nodeSpec = new NodeSpec.Builder() + .hostname(context.hostname().value()) + .nodeType(nodeType) + .state(Node.State.active) + .parentHostname("host123.test.domain.tld") + .owner(new NodeSpec.Owner("tenant", "application", "instance")) + .membership(new NodeSpec.Membership("clusterType", "clusterId", null, 0, false)) + .vespaVersion("6.305.12") + .flavor("d-2-8-50") + .canonicalFlavor("d-2-8-50") + .build(); + Path path = context.pathOnHostFromPathInNode("/etc/yamas-agent"); + uncheck(() -> Files.createDirectories(path)); + storageMaintainer.writeMetricsConfig(context, nodeSpec); + return path; + } - @Test - public void testNonExistingDiskUsed() throws IOException, InterruptedException { - StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, null); - long usedBytes = storageMaintainer.getDiskUsedInBytes(folder.getRoot().toPath().resolve("doesn't exist")); - assertEquals(0L, usedBytes); + private void assertCheckEnds(Path checkPath, String contentsEnd) { + String contents = new UnixPath(checkPath).readUtf8File(); + assertTrue(contents, contents.endsWith(contentsEnd)); + } + + private void assertChecks(Path checksPath, String... checkNames) { + List<String> expectedChecks = Stream.of(checkNames).sorted().collect(Collectors.toList()); + List<String> actualChecks = FileFinder.files(checksPath).stream() + .map(FileFinder.FileAttributes::filename) + .map(filename -> filename.replaceAll("\\.yaml$", "")) + .sorted() + .collect(Collectors.toList()); + assertEquals(expectedChecks, actualChecks); + } } - @Test - public void archive_container_data_test() throws IOException { - // Create some files in containers - FileSystem fileSystem = TestFileSystem.create(); - NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1"); - createNodeAgentContextAndContainerStorage(fileSystem, "container-2"); - - Path pathToArchiveDir = fileSystem.getPath("/home/docker/container-archive"); - Files.createDirectories(pathToArchiveDir); - - Path containerStorageRoot = context1.pathOnHostFromPathInNode("/").getParent(); - Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toSet()); - assertEquals(ImmutableSet.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive); - - - // Archive container-1 - StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, pathToArchiveDir); - storageMaintainer.archiveNodeStorage(context1); - - // container-1 should be gone from container-storage - Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot) - .maxDepth(1) - .stream() - .map(FileFinder.FileAttributes::filename) - .collect(Collectors.toSet()); - assertEquals(ImmutableSet.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive); - - // container archive directory should contain exactly 1 directory - the one we just archived - List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list(); - assertEquals(1, containerArchiveContentsAfterArchive.size()); - Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path(); - assertTrue(archivedContainerStoragePath.getFileName().toString().matches("container-1_[0-9]{14}")); - Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath) - .stream() - .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString()) - .collect(Collectors.toSet()); - assertEquals(ImmutableSet.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents); + public static class DiskUsageTests { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testDiskUsed() throws IOException, InterruptedException { + StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, null); + int writeSize = 10000; + Files.write(folder.newFile().toPath(), new byte[writeSize]); + + long usedBytes = storageMaintainer.getDiskUsedInBytes(folder.getRoot().toPath()); + if (usedBytes * 4 < writeSize || usedBytes > writeSize * 4) + fail("Used bytes is " + usedBytes + ", but wrote " + writeSize + " bytes, not even close."); + } + + @Test + public void testNonExistingDiskUsed() throws IOException, InterruptedException { + StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, null); + long usedBytes = storageMaintainer.getDiskUsedInBytes(folder.getRoot().toPath().resolve("doesn't exist")); + assertEquals(0L, usedBytes); + } } - private NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { - NodeAgentContext context = new NodeAgentContextImpl.Builder(containerName + ".domain.tld") - .fileSystem(fileSystem).build(); - - Path containerVespaHomeOnHost = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("")); - Files.createDirectories(context.pathOnHostFromPathInNode("/etc/something")); - Files.createFile(context.pathOnHostFromPathInNode("/etc/something/conf")); - - Files.createDirectories(containerVespaHomeOnHost.resolve("logs/vespa")); - Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/vespa.log")); - Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/zookeeper.log")); - - Files.createDirectories(containerVespaHomeOnHost.resolve("var/db")); - Files.createFile(containerVespaHomeOnHost.resolve("var/db/some-file")); - - Path containerRootOnHost = context.pathOnHostFromPathInNode("/"); - Set<String> actualContents = FileFinder.files(containerRootOnHost) - .stream() - .map(fileAttributes -> containerRootOnHost.relativize(fileAttributes.path()).toString()) - .collect(Collectors.toSet()); - Set<String> expectedContents = new HashSet<>(Arrays.asList( - "etc/something/conf", - "opt/vespa/logs/vespa/vespa.log", - "opt/vespa/logs/vespa/zookeeper.log", - "opt/vespa/var/db/some-file")); - assertEquals(expectedContents, actualContents); - return context; + public static class ArchiveContainerDataTests { + @Test + public void archive_container_data_test() throws IOException { + // Create some files in containers + FileSystem fileSystem = TestFileSystem.create(); + NodeAgentContext context1 = createNodeAgentContextAndContainerStorage(fileSystem, "container-1"); + createNodeAgentContextAndContainerStorage(fileSystem, "container-2"); + + Path pathToArchiveDir = fileSystem.getPath("/home/docker/container-archive"); + Files.createDirectories(pathToArchiveDir); + + Path containerStorageRoot = context1.pathOnHostFromPathInNode("/").getParent(); + Set<String> containerStorageRootContentsBeforeArchive = FileFinder.from(containerStorageRoot) + .maxDepth(1) + .stream() + .map(FileFinder.FileAttributes::filename) + .collect(Collectors.toSet()); + assertEquals(ImmutableSet.of("container-archive", "container-1", "container-2"), containerStorageRootContentsBeforeArchive); + + + // Archive container-1 + StorageMaintainer storageMaintainer = new StorageMaintainer(docker, null, pathToArchiveDir); + storageMaintainer.archiveNodeStorage(context1); + + // container-1 should be gone from container-storage + Set<String> containerStorageRootContentsAfterArchive = FileFinder.from(containerStorageRoot) + .maxDepth(1) + .stream() + .map(FileFinder.FileAttributes::filename) + .collect(Collectors.toSet()); + assertEquals(ImmutableSet.of("container-archive", "container-2"), containerStorageRootContentsAfterArchive); + + // container archive directory should contain exactly 1 directory - the one we just archived + List<FileFinder.FileAttributes> containerArchiveContentsAfterArchive = FileFinder.from(pathToArchiveDir).maxDepth(1).list(); + assertEquals(1, containerArchiveContentsAfterArchive.size()); + Path archivedContainerStoragePath = containerArchiveContentsAfterArchive.get(0).path(); + assertTrue(archivedContainerStoragePath.getFileName().toString().matches("container-1_[0-9]{14}")); + Set<String> archivedContainerStorageContents = FileFinder.files(archivedContainerStoragePath) + .stream() + .map(fileAttributes -> archivedContainerStoragePath.relativize(fileAttributes.path()).toString()) + .collect(Collectors.toSet()); + assertEquals(ImmutableSet.of("opt/vespa/logs/vespa/vespa.log", "opt/vespa/logs/vespa/zookeeper.log"), archivedContainerStorageContents); + } + + private NodeAgentContext createNodeAgentContextAndContainerStorage(FileSystem fileSystem, String containerName) throws IOException { + NodeAgentContext context = new NodeAgentContextImpl.Builder(containerName + ".domain.tld") + .fileSystem(fileSystem).build(); + + Path containerVespaHomeOnHost = context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("")); + Files.createDirectories(context.pathOnHostFromPathInNode("/etc/something")); + Files.createFile(context.pathOnHostFromPathInNode("/etc/something/conf")); + + Files.createDirectories(containerVespaHomeOnHost.resolve("logs/vespa")); + Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/vespa.log")); + Files.createFile(containerVespaHomeOnHost.resolve("logs/vespa/zookeeper.log")); + + Files.createDirectories(containerVespaHomeOnHost.resolve("var/db")); + Files.createFile(containerVespaHomeOnHost.resolve("var/db/some-file")); + + Path containerRootOnHost = context.pathOnHostFromPathInNode("/"); + Set<String> actualContents = FileFinder.files(containerRootOnHost) + .stream() + .map(fileAttributes -> containerRootOnHost.relativize(fileAttributes.path()).toString()) + .collect(Collectors.toSet()); + Set<String> expectedContents = new HashSet<>(Arrays.asList( + "etc/something/conf", + "opt/vespa/logs/vespa/vespa.log", + "opt/vespa/logs/vespa/zookeeper.log", + "opt/vespa/var/db/some-file")); + assertEquals(expectedContents, actualContents); + return context; + } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index a270585bcaa..07d3fab9534 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -20,9 +20,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.anyVararg; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -59,8 +58,8 @@ public class AclMaintainerTest { aclMaintainer.converge(); - verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg()); //we don;t have a ip4 address for the container so no redirect either - verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg()); + verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), any()); //we don;t have a ip4 address for the container so no redirect either + verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), any()); } @Test @@ -77,8 +76,8 @@ public class AclMaintainerTest { aclMaintainer.converge(); - verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg()); //we don;t have a ip4 address for the container so no redirect either - verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg()); + verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), any()); //we don;t have a ip4 address for the container so no redirect either + verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), any()); } @Test @@ -90,7 +89,7 @@ public class AclMaintainerTest { aclMaintainer.converge(); - verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), anyVararg()); + verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), any()); } @Test @@ -123,8 +122,8 @@ public class AclMaintainerTest { aclMaintainer.converge(); - verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), anyVararg()); - verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), anyVararg()); + verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(container.name), eq("iptables-restore"), any()); + verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(container.name), eq("ip6tables-restore"), any()); } @Test @@ -166,8 +165,8 @@ public class AclMaintainerTest { aclMaintainer.converge(); - verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("ip6tables-restore"), anyVararg()); - verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("iptables-restore"), anyVararg()); + verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("ip6tables-restore"), any()); + verify(dockerOperations, never()).executeCommandInNetworkNamespace(any(), eq("iptables-restore"), any()); } @@ -189,11 +188,11 @@ public class AclMaintainerTest { when(dockerOperations.executeCommandInNetworkNamespace( eq(container.name), - eq("ip6tables-restore"), anyVararg())).thenThrow(new RuntimeException("iptables restore failed")); + eq("ip6tables-restore"), any())).thenThrow(new RuntimeException("iptables restore failed")); when(dockerOperations.executeCommandInNetworkNamespace( eq(container.name), - eq("iptables-restore"), anyVararg())).thenThrow(new RuntimeException("iptables restore failed")); + eq("iptables-restore"), any())).thenThrow(new RuntimeException("iptables restore failed")); aclMaintainer.converge(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditorTest.java index 5496adf3e69..1b39014a9c6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditorTest.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; import org.junit.Assert; import org.junit.Test; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index ca8435c9edd..af9e7b39657 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -33,8 +33,8 @@ import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index bb229a10f59..3860e2e9780 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -21,9 +21,9 @@ import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java index 82f2408b252..6afeca6eeb5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterImplTest.java @@ -21,9 +21,9 @@ import java.util.stream.IntStream; import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdaterImpl.TRANSITION_EXCEPTION_MESSAGE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 00326f90caa..0e72ef15f6a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -38,9 +38,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyVararg; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; @@ -88,17 +87,12 @@ public class NodeAgentImplTest { @Test public void upToDateContainerIsUntouched() { - final long restartGeneration = 1; - final long rebootGeneration = 0; final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(Node.State.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) - .wantedRestartGeneration(restartGeneration) - .currentRestartGeneration(restartGeneration) - .wantedRebootGeneration(rebootGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -121,17 +115,12 @@ public class NodeAgentImplTest { @Test public void verifyRemoveOldFilesIfDiskFull() { - final long restartGeneration = 1; - final long rebootGeneration = 0; final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(Node.State.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) - .wantedRestartGeneration(restartGeneration) - .currentRestartGeneration(restartGeneration) - .wantedRebootGeneration(rebootGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -182,14 +171,12 @@ public class NodeAgentImplTest { @Test public void absentContainerCausesStart() { final Optional<Long> restartGeneration = Optional.of(1L); - final long rebootGeneration = 0; final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .state(Node.State.active) .wantedVespaVersion(vespaVersion) .wantedRestartGeneration(restartGeneration.get()) .currentRestartGeneration(restartGeneration.get()) - .wantedRebootGeneration(rebootGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(null, false); @@ -211,26 +198,19 @@ public class NodeAgentImplTest { inOrder.verify(aclMaintainer, times(1)).converge(); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes() - .withRestartGeneration(restartGeneration) - .withRebootGeneration(rebootGeneration) - .withDockerImage(dockerImage)); + hostName, new NodeAttributes().withDockerImage(dockerImage)); inOrder.verify(orchestrator).resume(hostName); } @Test public void containerIsNotStoppedIfNewImageMustBePulled() { final DockerImage newDockerImage = new DockerImage("new-image"); - final long wantedRestartGeneration = 2; - final long currentRestartGeneration = 1; final NodeSpec node = nodeBuilder .wantedDockerImage(newDockerImage) .currentDockerImage(dockerImage) .state(Node.State.active) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) - .wantedRestartGeneration(wantedRestartGeneration) - .currentRestartGeneration(currentRestartGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -251,16 +231,12 @@ public class NodeAgentImplTest { @Test public void containerIsRestartedIfFlavorChanged() { - final long wantedRestartGeneration = 1; - final long currentRestartGeneration = 1; NodeSpec.Builder specBuilder = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(Node.State.active) .wantedVespaVersion(vespaVersion) - .vespaVersion(vespaVersion) - .wantedRestartGeneration(wantedRestartGeneration) - .currentRestartGeneration(currentRestartGeneration); + .vespaVersion(vespaVersion); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); NodeSpec firstSpec = specBuilder.build(); @@ -316,16 +292,44 @@ public class NodeAgentImplTest { } @Test + public void recreatesContainerIfRebootWanted() { + final long wantedRebootGeneration = 2; + final long currentRebootGeneration = 1; + final NodeSpec node = nodeBuilder + .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) + .state(Node.State.active) + .wantedVespaVersion(vespaVersion) + .vespaVersion(vespaVersion) + .wantedRebootGeneration(wantedRebootGeneration) + .currentRebootGeneration(currentRebootGeneration) + .build(); + + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); + when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); + + nodeAgent.converge(); + + verify(orchestrator, times(1)).suspend(eq(hostName)); + verify(dockerOperations, times(1)).removeContainer(eq(context), any()); + verify(dockerOperations, times(1)).createContainer(eq(context), eq(node), any()); + verify(dockerOperations, times(1)).startContainer(eq(context)); + verify(orchestrator, times(1)).resume(eq(hostName)); + verify(nodeRepository, times(1)).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() + .withRebootGeneration(wantedRebootGeneration))); + } + + @Test public void failedNodeRunningContainerShouldStillBeRunning() { - final long restartGeneration = 1; final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(Node.State.failed) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) - .wantedRestartGeneration(restartGeneration) - .currentRestartGeneration(restartGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -341,13 +345,8 @@ public class NodeAgentImplTest { @Test public void readyNodeLeadsToNoAction() { - final long restartGeneration = 1; - final long rebootGeneration = 0; final NodeSpec node = nodeBuilder .state(Node.State.ready) - .wantedRestartGeneration(restartGeneration) - .currentRestartGeneration(restartGeneration) - .wantedRebootGeneration(rebootGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(null,false); @@ -369,18 +368,12 @@ public class NodeAgentImplTest { @Test public void inactiveNodeRunningContainerShouldStillBeRunning() { - final long restartGeneration = 1; - final long rebootGeneration = 0; - final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(Node.State.inactive) .wantedVespaVersion(vespaVersion) .vespaVersion(vespaVersion) - .wantedRestartGeneration(restartGeneration) - .currentRestartGeneration(restartGeneration) - .wantedRebootGeneration(rebootGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -398,16 +391,10 @@ public class NodeAgentImplTest { @Test public void reservedNodeDoesNotUpdateNodeRepoWithVersion() { - final long restartGeneration = 1; - final long rebootGeneration = 0; - final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .state(Node.State.reserved) .wantedVespaVersion(vespaVersion) - .wantedRestartGeneration(restartGeneration) - .currentRestartGeneration(restartGeneration) - .wantedRebootGeneration(rebootGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(null, false); @@ -451,10 +438,7 @@ public class NodeAgentImplTest { verify(orchestrator, never()).suspend(any(String.class)); // current Docker image and vespa version should be cleared verify(nodeRepository, times(1)).updateNodeAttributes( - any(String.class), eq(new NodeAttributes() - .withRestartGeneration(wantedRestartGeneration) - .withRebootGeneration(0L) - .withDockerImage(new DockerImage("")))); + eq(hostName), eq(new NodeAttributes().withDockerImage(new DockerImage("")))); } @Test @@ -504,14 +488,11 @@ public class NodeAgentImplTest { @Test public void resumeProgramRunsUntilSuccess() { - final long restartGeneration = 1; final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .currentDockerImage(dockerImage) .state(Node.State.active) .vespaVersion(vespaVersion) - .wantedRestartGeneration(restartGeneration) - .currentRestartGeneration(restartGeneration) .build(); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); @@ -570,15 +551,10 @@ public class NodeAgentImplTest { @Test public void start_container_subtask_failure_leads_to_container_restart() { - final long restartGeneration = 1; - final long rebootGeneration = 0; final NodeSpec node = nodeBuilder .wantedDockerImage(dockerImage) .state(Node.State.active) .wantedVespaVersion(vespaVersion) - .wantedRestartGeneration(restartGeneration) - .currentRestartGeneration(restartGeneration) - .wantedRebootGeneration(rebootGeneration) .build(); NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false)); @@ -669,7 +645,7 @@ public class NodeAgentImplTest { assertEquals(5L, calledTimeout); assertArrayEquals(expectedCommand, calledCommand); return null; - }).when(dockerOperations).executeCommandInContainerAsRoot(any(), any(), anyVararg()); + }).when(dockerOperations).executeCommandInContainerAsRoot(any(), any(), any()); nodeAgent.updateContainerNodeMetrics(); } @@ -695,7 +671,6 @@ public class NodeAgentImplTest { @Test public void testRunningConfigServer() { - final long rebootGeneration = 0; final NodeSpec node = nodeBuilder .nodeType(NodeType.config) .wantedDockerImage(dockerImage) @@ -721,9 +696,7 @@ public class NodeAgentImplTest { inOrder.verify(aclMaintainer, times(1)).converge(); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes() - .withRebootGeneration(rebootGeneration) - .withDockerImage(dockerImage)); + hostName, new NodeAttributes().withDockerImage(dockerImage)); inOrder.verify(orchestrator).resume(hostName); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java index 13d06e326c7..c325a858d7c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java @@ -13,7 +13,7 @@ import java.util.Collections; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java index 1e4c7482aaf..2bc64a3fdb3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriterTest.java @@ -13,8 +13,8 @@ import java.time.Instant; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java index 1a88af8ad0f..6151d16b69e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java @@ -16,8 +16,8 @@ import java.time.Instant; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java index e0ce75b6fc2..30263403757 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/SecretAgentCheckConfigTest.java @@ -36,7 +36,7 @@ public class SecretAgentCheckConfigTest { " interval: 60\n" + " user: nobody\n" + " check: /some/test\n" + - " args: \n" + + " args:\n" + " - arg1\n" + " - arg2 with space\n" + " tags:\n" + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index b0e1632002b..5060510be20 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -177,23 +177,22 @@ public class NodeRepository extends AbstractComponent { */ private NodeAcl getNodeAcl(Node node, NodeList candidates) { Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname)); - Set<String> trustedNetworks = new HashSet<>(); Set<Integer> trustedPorts = new HashSet<>(); // For all cases below, trust: // - nodes in same application - // - config servers // - ssh node.allocation().ifPresent(allocation -> trustedNodes.addAll(candidates.owner(allocation.owner()).asList())); - trustedNodes.addAll(candidates.nodeType(NodeType.config).asList()); trustedPorts.add(22); switch (node.type()) { case tenant: // Tenant nodes in other states than ready, trust: + // - config servers // - proxy nodes // - parent (Docker) hosts of already trusted nodes. This is needed in a transition period, while // we migrate away from IPv4-only nodes + trustedNodes.addAll(candidates.nodeType(NodeType.config).asList()); trustedNodes.addAll(candidates.parentsOf(trustedNodes).asList()); // TODO: Remove when we no longer have IPv4-only nodes trustedNodes.addAll(candidates.nodeType(NodeType.proxy).asList()); if (node.state() == Node.State.ready) { @@ -206,24 +205,27 @@ public class NodeRepository extends AbstractComponent { break; case config: - // Config servers trust all nodes + // Config servers trust: + // - all nodes + // - port 4443 from the world trustedNodes.addAll(candidates.asList()); - - // And all connections on 4443 trustedPorts.add(4443); break; case proxy: - // Accept connections from the world on 443 (for dashboard app), 4080 (insecure tb removed), and 4443 + // Proxy nodes trust: + // - config servers + // - all connections from the world on 443 (for dashboard app), 4080 (insecure tb removed), and 4443 + trustedNodes.addAll(candidates.nodeType(NodeType.config).asList()); trustedPorts.add(443); trustedPorts.add(4080); trustedPorts.add(4443); break; - case host: - // This is only needed for macvlan networks - for nated networks this is handled elsewhere. - // Docker bridge network - trustedNetworks.add("172.17.0.0/16"); + case controller: + // Controllers: + // - port 4443 (HTTPS) from the world + trustedPorts.add(4443); break; default: @@ -232,7 +234,7 @@ public class NodeRepository extends AbstractComponent { node.hostname(), node.type())); } - return new NodeAcl(node, trustedNodes, trustedNetworks, trustedPorts); + return new NodeAcl(node, trustedNodes, Collections.emptySet(), trustedPorts); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java index fd05eb86667..70750dd6672 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java @@ -72,4 +72,5 @@ public class OrchestratorMock implements Orchestrator { public void suspendAll(HostName parentHostname, List<HostName> hostNames) { hostNames.forEach(this::suspend); } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index c5f33800283..5d8bde960d8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; @@ -26,6 +27,7 @@ import static com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester.c import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author mpolden @@ -34,8 +36,6 @@ public class AclProvisioningTest { private ProvisioningTester tester; - private final List<String> dockerBridgeNetwork = Collections.singletonList("172.17.0.0/16"); - @Before public void before() { this.tester = new ProvisioningTester(Zone.defaultZone(), createConfig()); @@ -126,45 +126,7 @@ public class AclProvisioningTest { } @Test - public void trusted_nodes_for_docker_host() { - List<Node> configServers = tester.makeConfigServers(3, "default", Version.fromString("6.123.456")); - - // Populate repo - tester.makeReadyNodes(2, "default", NodeType.host); - - // Deploy zone application - ApplicationId zoneApplication = tester.makeApplicationId(); - allocateNodes(Capacity.fromRequiredNodeType(NodeType.host), zoneApplication); - - List<Node> dockerHostNodes = tester.nodeRepository().getNodes(zoneApplication); - List<NodeAcl> acls = tester.nodeRepository().getNodeAcls(dockerHostNodes.get(0), false); - - // Trusted nodes is all Docker hosts and all config servers - assertAcls(Arrays.asList(dockerHostNodes, configServers), dockerBridgeNetwork, acls.get(0)); - } - - - @Test - public void trusted_nodes_for_docker_hosts_nodes_in_zone_application() { - List<Node> configServers = tester.makeConfigServers(3, "default", Version.fromString("6.123.456")); - ApplicationId applicationId = tester.makeApplicationId(); // use same id for both allocate calls below - - // Populate repo - tester.makeReadyNodes(2, "default", NodeType.host); - - // Allocate 2 Docker hosts - List<Node> activeDockerHostNodes = allocateNodes(NodeType.host, applicationId); - assertEquals(2, activeDockerHostNodes.size()); - - // Check trusted nodes for all nodes - activeDockerHostNodes.forEach(node -> { - List<NodeAcl> nodeAcls = tester.nodeRepository().getNodeAcls(node, false); - assertAcls(Arrays.asList(activeDockerHostNodes, configServers), dockerBridgeNetwork, nodeAcls); - }); - } - - @Test - public void trusted_nodes_for_child_nodes_of_docker_host() { + public void trusted_nodes_for_children_of_docker_host() { List<Node> configServers = tester.makeConfigServers(3, "default", Version.fromString("6.123.456")); // Populate repo @@ -189,6 +151,20 @@ public class AclProvisioningTest { } @Test + public void trusted_nodes_for_controllers() { + tester.makeReadyNodes(3, "default", NodeType.controller); + + // Allocate + ApplicationId controllerApplication = tester.makeApplicationId(); + List<Node> controllers = allocateNodes(Capacity.fromRequiredNodeType(NodeType.controller), controllerApplication); + + // Controllers and hosts all trust each other + List<NodeAcl> controllerAcls = tester.nodeRepository().getNodeAcls(controllers.get(0), false); + assertAcls(Collections.singletonList(controllers), controllerAcls); + assertEquals(ImmutableSet.of(22, 4443), controllerAcls.get(0).trustedPorts()); + } + + @Test public void resolves_hostnames_from_connection_spec() { tester.makeConfigServers(3, "default", Version.fromString("6.123.456")); @@ -206,10 +182,6 @@ public class AclProvisioningTest { return allocateNodes(Capacity.fromNodeCount(nodeCount), tester.makeApplicationId()); } - private List<Node> allocateNodes(NodeType nodeType, ApplicationId applicationId) { - return allocateNodes(Capacity.fromRequiredNodeType(nodeType), applicationId); - } - private List<Node> allocateNodes(Capacity capacity, ApplicationId applicationId) { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); @@ -219,18 +191,10 @@ public class AclProvisioningTest { } private static void assertAcls(List<List<Node>> expected, NodeAcl actual) { - assertAcls(expected, Collections.emptyList(), Collections.singletonList(actual)); - } - - private static void assertAcls(List<List<Node>> expected, List<NodeAcl> actual) { - assertAcls(expected, Collections.emptyList(), actual); - } - - private static void assertAcls(List<List<Node>> expected, List<String> expectedNetworks, NodeAcl actual) { - assertAcls(expected, expectedNetworks, Collections.singletonList(actual)); + assertAcls(expected, Collections.singletonList(actual)); } - private static void assertAcls(List<List<Node>> expectedNodes, List<String> expectedNetworks, List<NodeAcl> actual) { + private static void assertAcls(List<List<Node>> expectedNodes, List<NodeAcl> actual) { Set<Node> expectedTrustedNodes = expectedNodes.stream() .flatMap(Collection::stream) .collect(Collectors.toSet()); @@ -239,10 +203,9 @@ public class AclProvisioningTest { .collect(Collectors.toSet()); assertEquals(expectedTrustedNodes, actualTrustedNodes); - Set<String> expectedTrustedNetworks = new HashSet<>(expectedNetworks); Set<String> actualTrustedNetworks = actual.stream() .flatMap(acl -> acl.trustedNetworks().stream()) .collect(Collectors.toSet()); - assertEquals(expectedTrustedNetworks, actualTrustedNetworks); + assertTrue("No networks are trusted", actualTrustedNetworks.isEmpty()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index ebcc46d5661..2ff1e403e35 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -354,7 +354,7 @@ public class RestApiTest { @Test public void acl_request_by_docker_host() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/acl/dockerhost1.yahoo.com"), "acl-docker-host.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/acl/dockerhost1.yahoo.com?children=true"), "acl-docker-host.json"); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-docker-host.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-docker-host.json index 4d6607bd1b0..abf3c39001f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-docker-host.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-docker-host.json @@ -4,85 +4,62 @@ "hostname": "cfg1.yahoo.com", "type": "config", "ipAddress": "127.0.1.1", - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" }, { "hostname": "cfg2.yahoo.com", "type": "config", "ipAddress": "127.0.1.2", - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" }, { "hostname": "dockerhost1.yahoo.com", "type": "host", "ipAddress": "::1", - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" }, { "hostname": "dockerhost1.yahoo.com", "type": "host", "ipAddress": "127.0.0.1", - "trustedBy": "dockerhost1.yahoo.com" - }, - { - "hostname": "dockerhost2.yahoo.com", - "type": "host", - "ipAddress": "::1", - "trustedBy": "dockerhost1.yahoo.com" - }, - { - "hostname": "dockerhost2.yahoo.com", - "type": "host", - "ipAddress": "127.0.0.1", - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" }, { "hostname": "dockerhost3.yahoo.com", "type": "host", "ipAddress": "::1", - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" }, { "hostname": "dockerhost3.yahoo.com", "type": "host", "ipAddress": "127.0.0.1", - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" }, { - "hostname": "dockerhost4.yahoo.com", - "type": "host", + "hostname": "host4.yahoo.com", + "type": "tenant", "ipAddress": "::1", - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" }, { - "hostname": "dockerhost4.yahoo.com", - "type": "host", + "hostname": "host4.yahoo.com", + "type": "tenant", "ipAddress": "127.0.0.1", - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" }, { - "hostname": "dockerhost5.yahoo.com", - "type": "host", - "ipAddress": "::1", - "trustedBy": "dockerhost1.yahoo.com" - }, - { - "hostname": "dockerhost5.yahoo.com", - "type": "host", - "ipAddress": "127.0.0.1", - "trustedBy": "dockerhost1.yahoo.com" - } - ], - "trustedNetworks": [ - { - "network": "172.17.0.0/16", - "trustedBy": "dockerhost1.yahoo.com" + "hostname": "test-container-1", + "type": "tenant", + "ipAddress": "::2", + "trustedBy": "host4.yahoo.com" } ], + "trustedNetworks": [], "trustedPorts": [ { "port": 22, - "trustedBy": "dockerhost1.yahoo.com" + "trustedBy": "host4.yahoo.com" } ] } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java index e37f5ad4d70..df124f2f690 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java @@ -33,6 +33,7 @@ public interface Orchestrator { /** * Get orchestrator information related to a host. + * * @throws HostNameNotFoundException */ Host getHost(HostName hostName) throws HostNameNotFoundException; @@ -110,4 +111,5 @@ public interface Orchestrator { * @param appId Identifier of the application to resume */ void suspend(ApplicationId appId) throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException; + } diff --git a/orchestrator/src/test/application/services.xml b/orchestrator/src/test/application/services.xml deleted file mode 100644 index 063e0f3e0cc..00000000000 --- a/orchestrator/src/test/application/services.xml +++ /dev/null @@ -1,16 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services> - <jdisc version="1.0" jetty="true"> - <config name="container.handler.threadpool"> - <maxthreads>10</maxthreads> - </config> - <component id="com.yahoo.vespa.orchestrator.status.InMemoryStatusService" bundle="orchestrator" /> - <component id="com.yahoo.vespa.orchestrator.DummyInstanceLookupService" bundle="orchestrator" /> - <component id="com.yahoo.vespa.orchestrator.OrchestratorImpl" bundle="orchestrator" /> - <component id="com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock" bundle="orchestrator" /> - - <rest-api path="orchestrator" jersey2="true"> - <components bundle="orchestrator" /> - </rest-api> - </jdisc> -</services> diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java index 8595c8a31a4..c575811fd6d 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java @@ -17,31 +17,31 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.io.IOException; +import java.net.ServerSocket; import java.net.URI; -import java.nio.file.Paths; import java.util.Set; import static org.junit.Assert.assertEquals; /** - * Tests the implementation of the orchestrators ApplicationAPI. + * Tests the implementation of the orchestrator Application API. * * @author smorgrav */ public class ApplicationSuspensionResourceTest { - static final String BASE_PATH = "/orchestrator/v1/suspensions/applications"; - static final String RESOURCE_1 = "mediasearch:imagesearch:default"; - static final String RESOURCE_2 = "test-tenant-id:application:instance"; - static final String INVALID_RESOURCE_NAME = "something_without_colons"; + private static final String BASE_PATH = "/orchestrator/v1/suspensions/applications"; + private static final String RESOURCE_1 = "mediasearch:imagesearch:default"; + private static final String RESOURCE_2 = "test-tenant-id:application:instance"; + private static final String INVALID_RESOURCE_NAME = "something_without_colons"; - Application jdiscApplication; - WebTarget webTarget; + private Application jdiscApplication; + private WebTarget webTarget; @Before public void setup() throws Exception { - jdiscApplication = Application.fromApplicationPackage(Paths.get("src/test/application"), - Networking.enable); + jdiscApplication = Application.fromServicesXml(servicesXml(), Networking.enable); Client client = ClientBuilder.newClient(); JettyHttpServer serverProvider = (JettyHttpServer) Container.get().getServerProviderRegistry().allComponents().get(0); @@ -50,7 +50,7 @@ public class ApplicationSuspensionResourceTest { } @After - public void teardown() throws Exception { + public void teardown() { jdiscApplication.close(); webTarget = null; } @@ -63,26 +63,26 @@ public class ApplicationSuspensionResourceTest { } @Test - public void get_all_suspended_applications_return_empty_list_initially() throws Exception { + public void get_all_suspended_applications_return_empty_list_initially() { Response reply = webTarget.request().get(); assertEquals(200, reply.getStatus()); assertEquals("[]", reply.readEntity(String.class)); } @Test - public void invalid_application_id_throws_http_400() throws Exception { - Response reply = webTarget.request().post(Entity.entity(INVALID_RESOURCE_NAME,MediaType.APPLICATION_JSON_TYPE)); + public void invalid_application_id_throws_http_400() { + Response reply = webTarget.request().post(Entity.entity(INVALID_RESOURCE_NAME, MediaType.APPLICATION_JSON_TYPE)); assertEquals(400, reply.getStatus()); } @Test - public void get_application_status_returns_404_for_notsuspended_and_204_for_suspended() throws Exception { + public void get_application_status_returns_404_for_not_suspended_and_204_for_suspended() { // Get on application that is not suspended Response reply = webTarget.path(RESOURCE_1).request().get(); assertEquals(404, reply.getStatus()); // Post application - reply = webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE)); + reply = webTarget.request().post(Entity.entity(RESOURCE_1, MediaType.APPLICATION_JSON_TYPE)); assertEquals(204, reply.getStatus()); // Get on the application that now should be in suspended @@ -90,15 +90,14 @@ public class ApplicationSuspensionResourceTest { assertEquals(204, reply.getStatus()); } - @Test - public void delete_works_on_suspended_and_not_suspended_applications() throws Exception { + public void delete_works_on_suspended_and_not_suspended_applications() { // Delete an application that is not suspended Response reply = webTarget.path(RESOURCE_1).request().delete(); assertEquals(204, reply.getStatus()); // Put application in suspend - reply = webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE)); + reply = webTarget.request().post(Entity.entity(RESOURCE_1, MediaType.APPLICATION_JSON_TYPE)); assertEquals(204, reply.getStatus()); // Check that it is in suspend @@ -115,15 +114,15 @@ public class ApplicationSuspensionResourceTest { } @Test - public void list_applications_returns_the_correct_list_of_suspended_applications() throws Exception { + public void list_applications_returns_the_correct_list_of_suspended_applications() { // Test that initially we have the empty set Response reply = webTarget.request(MediaType.APPLICATION_JSON).get(); assertEquals(200, reply.getStatus()); assertEquals("[]", reply.readEntity(String.class)); // Add a couple of applications to maintenance - webTarget.request().post(Entity.entity(RESOURCE_1,MediaType.APPLICATION_JSON_TYPE)); - webTarget.request().post(Entity.entity(RESOURCE_2,MediaType.APPLICATION_JSON_TYPE)); + webTarget.request().post(Entity.entity(RESOURCE_1, MediaType.APPLICATION_JSON_TYPE)); + webTarget.request().post(Entity.entity(RESOURCE_2, MediaType.APPLICATION_JSON_TYPE)); assertEquals(200, reply.getStatus()); // Test that we get them back @@ -140,4 +139,39 @@ public class ApplicationSuspensionResourceTest { assertEquals(1, responses.size()); assertEquals(RESOURCE_2, responses.iterator().next()); } + + private String servicesXml() { + int port = 0; + do { + try { + ServerSocket socket = new ServerSocket(0); + port = socket.getLocalPort(); + socket.close(); + System.out.println("Using port " + port); + } catch (IOException e) { + // ignore + } + } while (port == 0); + + return "<services>\n" + + " <jdisc version=\"1.0\" jetty=\"true\">\n" + + " <config name=\"container.handler.threadpool\">\n" + + " <maxthreads>10</maxthreads>\n" + + " </config>\n" + + " <component id=\"com.yahoo.vespa.orchestrator.status.InMemoryStatusService\" bundle=\"orchestrator\" />\n" + + " <component id=\"com.yahoo.vespa.orchestrator.DummyInstanceLookupService\" bundle=\"orchestrator\" />\n" + + " <component id=\"com.yahoo.vespa.orchestrator.OrchestratorImpl\" bundle=\"orchestrator\" />\n" + + " <component id=\"com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock\" bundle=\"orchestrator\" />\n" + + "\n" + + " <rest-api path=\"orchestrator\">\n" + + " <components bundle=\"orchestrator\" />\n" + + " </rest-api>\n" + + "\n" + + " <http>\n" + + " <server id=\"foo\" port=\"" + port + "\"/>\n" + + " </http>\n" + + " </jdisc>\n" + + "</services>\n"; + } + } diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ApplicationView.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ApplicationView.java index 00503212750..c3fb3616ca9 100644 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ApplicationView.java +++ b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ApplicationView.java @@ -6,8 +6,10 @@ import java.util.List; /** * All clusters of a deployed application. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class ApplicationView { + public List<ClusterView> clusters; + } diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index 0ff648e271c..e38475e65c8 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -51,7 +51,8 @@ <!-- compile --> <dependency> <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zms-java-client</artifactId> + <artifactId>athenz-auth-core</artifactId> + <version>${athenz.version}</version> <scope>compile</scope> <exclusions> <exclusion> @@ -62,39 +63,6 @@ <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk15on</artifactId> </exclusion> - <!-- Exclude all Jersey bundles provided by JDisc --> - <exclusion> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-client</artifactId> - </exclusion> - <exclusion> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-json-jackson</artifactId> - </exclusion> - <!--Exclude all Jackson bundles provided by JDisc --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zts-java-client</artifactId> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> <!--Exclude all Jackson bundles provided by JDisc --> <exclusion> <groupId>com.fasterxml.jackson.core</groupId> diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzResourceGroup.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzResourceGroup.java new file mode 100644 index 00000000000..2825cf57c7b --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzResourceGroup.java @@ -0,0 +1,40 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.api; + +import java.util.Objects; + +/** + * @author bjorncs + */ +public class AthenzResourceGroup { + private final String name; + + public AthenzResourceGroup(String name) { + this.name = name; + } + + public String name() { + return name; + } + + @Override + public String toString() { + return "AthenzResourceGroup{" + + "name='" + name + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AthenzResourceGroup that = (AthenzResourceGroup) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/OktaAccessToken.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/OktaAccessToken.java new file mode 100644 index 00000000000..8b19f7abdd5 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/OktaAccessToken.java @@ -0,0 +1,42 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.api; + +import java.util.Objects; + +/** + * @author bjorncs + */ +public class OktaAccessToken { + + public static final String HTTP_HEADER_NAME = "Okta-Access-Token"; + + private final String token; + + public OktaAccessToken(String token) { + this.token = token; + } + + public String token() { + return token; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OktaAccessToken that = (OktaAccessToken) o; + return Objects.equals(token, that.token); + } + + @Override + public int hashCode() { + return Objects.hash(token); + } + + @Override + public String toString() { + return "OktaAccessToken{" + + "token='" + token + '\'' + + '}'; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java new file mode 100644 index 00000000000..7ff9db327d3 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java @@ -0,0 +1,99 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.common; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.yahoo.vespa.athenz.client.common.bindings.ErrorResponseEntity; +import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory; +import org.apache.http.HttpResponse; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; +import org.apache.http.impl.client.HttpClientBuilder; +import org.eclipse.jetty.http.HttpStatus; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.time.Duration; +import java.util.function.Supplier; + +/** + * @author bjorncs + */ +public abstract class ClientBase implements AutoCloseable { + + private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); + + private final CloseableHttpClient client; + private final ClientExceptionFactory exceptionFactory; + + protected ClientBase(String userAgent, + Supplier<SSLContext> sslContextSupplier, + ClientExceptionFactory exceptionFactory) { + this.exceptionFactory = exceptionFactory; + this.client = createHttpClient(userAgent, sslContextSupplier); + } + + protected <T> T execute(HttpUriRequest request, ResponseHandler<T> responseHandler) { + try { + return client.execute(request, responseHandler); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + protected StringEntity toJsonStringEntity(Object entity) { + try { + return new StringEntity(objectMapper.writeValueAsString(entity), ContentType.APPLICATION_JSON); + } catch (JsonProcessingException e) { + throw new UncheckedIOException(e); + } + } + + protected <T> T readEntity(HttpResponse response, Class<T> entityType) throws IOException { + if (HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) { + if (entityType.equals(Void.class)) { + return null; + } else { + return objectMapper.readValue(response.getEntity().getContent(), entityType); + } + } else { + ErrorResponseEntity errorEntity = objectMapper.readValue(response.getEntity().getContent(), ErrorResponseEntity.class); + throw exceptionFactory.createException(errorEntity.code, errorEntity.description); + } + } + + private static CloseableHttpClient createHttpClient(String userAgent, Supplier<SSLContext> sslContextSupplier) { + return HttpClientBuilder.create() + .setRetryHandler(new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true)) + .setUserAgent(userAgent) + .setSSLSocketFactory(new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(sslContextSupplier), (HostnameVerifier)null)) + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout((int) Duration.ofSeconds(10).toMillis()) + .setConnectionRequestTimeout((int)Duration.ofSeconds(10).toMillis()) + .setSocketTimeout((int)Duration.ofSeconds(20).toMillis()) + .build()) + .build(); + } + + @Override + public void close() { + try { + this.client.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + protected interface ClientExceptionFactory { + RuntimeException createException(int errorCode, String description); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/ErrorResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/bindings/ErrorResponseEntity.java index 431af084f9f..acbb831194e 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/ErrorResponseEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/bindings/ErrorResponseEntity.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.client.zts.bindings; +package com.yahoo.vespa.athenz.client.common.bindings; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/Pkcs10CsrSerializer.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/serializers/Pkcs10CsrSerializer.java index ca33962c7c8..9b26f5f2517 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/Pkcs10CsrSerializer.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/serializers/Pkcs10CsrSerializer.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.client.zts.bindings.serializers; +package com.yahoo.vespa.athenz.client.common.serializers; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateDeserializer.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/serializers/X509CertificateDeserializer.java index 59f10a78a58..f948115d4f2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateDeserializer.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/serializers/X509CertificateDeserializer.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.client.zts.bindings.serializers; +package com.yahoo.vespa.athenz.client.common.serializers; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateListDeserializer.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/serializers/X509CertificateListDeserializer.java index 64b23af9295..baa56512a62 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/X509CertificateListDeserializer.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/serializers/X509CertificateListDeserializer.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.client.zts.bindings.serializers; +package com.yahoo.vespa.athenz.client.common.serializers; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/serializers/package-info.java index 4c442617494..f1fe2a75b1b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/serializers/package-info.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/serializers/package-info.java @@ -3,6 +3,6 @@ * @author bjorncs */ @ExportPackage -package com.yahoo.vespa.athenz.client.zts.bindings.serializers; +package com.yahoo.vespa.athenz.client.common.serializers; import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java new file mode 100644 index 00000000000..b68dc2dd758 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -0,0 +1,138 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms; + +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; +import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.OktaAccessToken; +import com.yahoo.vespa.athenz.client.common.ClientBase; +import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.DomainListResponseEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.MembershipResponseEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.ProviderResourceGroupRolesRequestEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.TenancyRequestEntity; +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import org.apache.http.Header; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.message.BasicHeader; + +import javax.net.ssl.SSLContext; +import java.net.URI; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; + +import static java.util.stream.Collectors.toList; + +/** + * @author bjorncs + */ +public class DefaultZmsClient extends ClientBase implements ZmsClient { + + private final URI zmsUrl; + private final AthenzIdentity identity; + + public DefaultZmsClient(URI zmsUrl, AthenzIdentity identity, SSLContext sslContext) { + this(zmsUrl, identity, () -> sslContext); + } + + public DefaultZmsClient(URI zmsUrl, ServiceIdentityProvider identityProvider) { + this(zmsUrl, identityProvider.identity(), identityProvider::getIdentitySslContext); + } + + private DefaultZmsClient(URI zmsUrl, AthenzIdentity identity, Supplier<SSLContext> sslContextSupplier) { + super("vespa-zms-client", sslContextSupplier, ZmsClientException::new); + this.zmsUrl = addTrailingSlash(zmsUrl); + this.identity = identity; + } + + private static URI addTrailingSlash(URI zmsUrl) { + return zmsUrl.getPath().endsWith("/") ? zmsUrl : URI.create(zmsUrl.toString() + '/'); + } + + @Override + public void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + URI uri = zmsUrl.resolve(String.format("domain/%s/tenancy/%s", tenantDomain.getName(), providerService.getFullName())); + HttpUriRequest request = RequestBuilder.put() + .setUri(uri) + .addHeader(creatOktaAccessTokenHeader(token)) + .setEntity(toJsonStringEntity(new TenancyRequestEntity(tenantDomain, providerService, Collections.emptyList()))) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + + @Override + public void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + URI uri = zmsUrl.resolve(String.format("domain/%s/tenancy/%s", tenantDomain.getName(), providerService.getFullName())); + HttpUriRequest request = RequestBuilder.delete() + .setUri(uri) + .addHeader(creatOktaAccessTokenHeader(token)) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + + @Override + public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { + URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); + HttpUriRequest request = RequestBuilder.put() + .setUri(uri) + .addHeader(creatOktaAccessTokenHeader(token)) + .setEntity(toJsonStringEntity(new ProviderResourceGroupRolesRequestEntity(providerService, tenantDomain, roleActions, resourceGroup))) + .build(); + execute(request, response -> readEntity(response, Void.class)); // Note: The ZMS API will actually return a json object that is similar to ProviderResourceGroupRolesRequestEntity + } + + @Override + public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token) { + URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); + HttpUriRequest request = RequestBuilder.delete() + .setUri(uri) + .addHeader(creatOktaAccessTokenHeader(token)) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + + @Override + public boolean getMembership(AthenzRole role, AthenzIdentity identity) { + URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s", role.domain().getName(), role.roleName(), identity.getFullName())); + HttpUriRequest request = RequestBuilder.get() + .setUri(uri) + .build(); + return execute(request, response -> { + MembershipResponseEntity membership = readEntity(response, MembershipResponseEntity.class); + return membership.isMember; + }); + } + + @Override + public List<AthenzDomain> getDomainList(String prefix) { + HttpUriRequest request = RequestBuilder.get() + .setUri(zmsUrl.resolve("domain")) + .addParameter("prefix", prefix) + .build(); + return execute(request, response -> { + DomainListResponseEntity result = readEntity(response, DomainListResponseEntity.class); + return result.domains.stream().map(AthenzDomain::new).collect(toList()); + }); + } + + @Override + public boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity) { + URI uri = zmsUrl.resolve(String.format("access/%s/%s", action, resource.toResourceNameString())); + HttpUriRequest request = RequestBuilder.get() + .setUri(uri) + .build(); + return execute(request, response -> { + AccessResponseEntity result = readEntity(response, AccessResponseEntity.class); + return result.granted; + }); + } + + private static Header creatOktaAccessTokenHeader(OktaAccessToken token) { + return new BasicHeader("Cookie", String.format("okta_at=%s", token.token())); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/RoleAction.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/RoleAction.java new file mode 100644 index 00000000000..405dd1aa56a --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/RoleAction.java @@ -0,0 +1,49 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms; + +import com.yahoo.vespa.athenz.api.AthenzRole; + +import java.util.Objects; + +/** + * @author bjorncs + */ +public class RoleAction { + private final String roleName; + private final String action; + + public RoleAction(String roleName, String action) { + this.roleName = roleName; + this.action = action; + } + + public String getRoleName() { + return roleName; + } + + public String getAction() { + return action; + } + + @Override + public String toString() { + return "RoleAction{" + + "roleName=" + roleName + + ", action='" + action + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoleAction that = (RoleAction) o; + return Objects.equals(roleName, that.roleName) && + Objects.equals(action, that.action); + } + + @Override + public int hashCode() { + return Objects.hash(roleName, action); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java new file mode 100644 index 00000000000..cf044edeac0 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -0,0 +1,35 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms; + +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; +import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.OktaAccessToken; + +import java.time.Instant; +import java.util.List; +import java.util.Set; + +/** + * @author bjorncs + */ +public interface ZmsClient extends AutoCloseable { + + void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token); + + void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token); + + void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token); + + void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token); + + boolean getMembership(AthenzRole role, AthenzIdentity identity); + + List<AthenzDomain> getDomainList(String prefix); + + boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity); + + void close(); +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClientException.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClientException.java new file mode 100644 index 00000000000..f1b3ab8e7da --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClientException.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms; + +/** + * An exception that can be thrown by {@link ZmsClient} implementations. + * + * @author bjorncs + */ +public class ZmsClientException extends RuntimeException { + + private final int errorCode; + private final String description; + + public ZmsClientException(int errorCode, String description) { + super(createMessage(errorCode, description)); + this.errorCode = errorCode; + this.description = description; + } + + public int getErrorCode() { + return errorCode; + } + + public String getDescription() { + return description; + } + + private static String createMessage(int code, String description) { + return String.format("Received error from ZMS: code=%d, message=\"%s\"", code, description); + } + +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AccessResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AccessResponseEntity.java new file mode 100644 index 00000000000..dcc17bc807a --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/AccessResponseEntity.java @@ -0,0 +1,19 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AccessResponseEntity { + public final boolean granted; + + @JsonCreator + public AccessResponseEntity(@JsonProperty("granted") boolean granted) { + this.granted = granted; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/DomainListResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/DomainListResponseEntity.java new file mode 100644 index 00000000000..938d85dd74f --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/DomainListResponseEntity.java @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DomainListResponseEntity { + public final List<String> domains; + + @JsonCreator + public DomainListResponseEntity(@JsonProperty("names") List<String> domains) { + this.domains = domains; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipResponseEntity.java new file mode 100644 index 00000000000..499afb48f25 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/MembershipResponseEntity.java @@ -0,0 +1,28 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class MembershipResponseEntity { + public final String memberName; + public final boolean isMember; + public final String roleName; + public final String expiration; + + @JsonCreator + public MembershipResponseEntity(@JsonProperty("memberName") String memberName, + @JsonProperty("isMember") boolean isMember, + @JsonProperty("roleName") String roleName, + @JsonProperty("expiration") String expiration) { + this.memberName = memberName; + this.isMember = isMember; + this.roleName = roleName; + this.expiration = expiration; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java new file mode 100644 index 00000000000..dccd18fed61 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java @@ -0,0 +1,56 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.client.zms.RoleAction; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toList; + +/** + * @author bjorncs + */ +public class ProviderResourceGroupRolesRequestEntity { + + @JsonProperty("domain") + private final String domain; + + @JsonProperty("service") + private final String service; + + @JsonProperty("tenant") + private final String tenant; + + @JsonProperty("roles") + private final List<TenantRoleAction> roles; + + @JsonProperty("resourceGroup") + private final String resourceGroup; + + public ProviderResourceGroupRolesRequestEntity(AthenzService providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { + this.domain = providerService.getDomainName(); + this.service = providerService.getName(); + this.tenant = tenantDomain.getName(); + this.roles = rolesActions.stream().map(roleAction -> new TenantRoleAction(roleAction.getRoleName(), roleAction.getAction())).collect(toList()); + this.resourceGroup = resourceGroup; + } + + public static class TenantRoleAction { + @JsonProperty("role") + private final String role; + + @JsonProperty("action") + private final String action; + + public TenantRoleAction(String role, String action) { + this.role = role; + this.action = action; + } + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java new file mode 100644 index 00000000000..7883a505c71 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java @@ -0,0 +1,31 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzService; + +import java.util.List; + +/** + * @author bjorncs + */ +public class TenancyRequestEntity { + + @JsonProperty("domain") + private final String tenantDomain; + + @JsonProperty("service") + private final String providerService; + + @JsonProperty("resourceGroups") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + private final List<String> resourceGroups; + + public TenancyRequestEntity(AthenzDomain tenantDomain, AthenzService providerService, List<String> resourceGroups) { + this.tenantDomain = tenantDomain.getName(); + this.providerService = providerService.getFullName(); + this.resourceGroups = resourceGroups; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/package-info.java index 15cfa99c749..c7e8297fe7f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/package-info.java @@ -1,5 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ @ExportPackage -package com.yahoo.vespa.hosted.node.admin.config; +package com.yahoo.vespa.athenz.client.zms; -import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index e9aba31cf56..9eef2ff9903 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -1,16 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zts; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.api.ZToken; -import com.yahoo.vespa.athenz.client.zts.bindings.ErrorResponseEntity; +import com.yahoo.vespa.athenz.client.common.ClientBase; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.InstanceIdentityCredentials; @@ -22,25 +19,13 @@ import com.yahoo.vespa.athenz.client.zts.bindings.RoleTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.TenantDomainsResponseEntity; import com.yahoo.vespa.athenz.client.zts.utils.IdentityCsrGenerator; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.vespa.athenz.identity.ServiceIdentitySslSocketFactory; import com.yahoo.security.Pkcs10Csr; import org.apache.http.HttpResponse; -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.client.HttpClientBuilder; -import org.eclipse.jetty.http.HttpStatus; - -import javax.net.ssl.HostnameVerifier; + import javax.net.ssl.SSLContext; import java.io.IOException; -import java.io.UncheckedIOException; import java.net.URI; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -56,12 +41,9 @@ import static java.util.stream.Collectors.toList; * @author bjorncs * @author mortent */ -public class DefaultZtsClient implements ZtsClient { - - private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); +public class DefaultZtsClient extends ClientBase implements ZtsClient { private final URI ztsUrl; - private final CloseableHttpClient client; private final AthenzIdentity identity; public DefaultZtsClient(URI ztsUrl, AthenzIdentity identity, SSLContext sslContext) { @@ -73,9 +55,9 @@ public class DefaultZtsClient implements ZtsClient { } private DefaultZtsClient(URI ztsUrl, AthenzIdentity identity, Supplier<SSLContext> sslContextSupplier) { + super("vespa-zts-client", sslContextSupplier, ZtsClientException::new); this.ztsUrl = addTrailingSlash(ztsUrl); this.identity = identity; - this.client = createHttpClient(sslContextSupplier); } @Override @@ -91,7 +73,7 @@ public class DefaultZtsClient implements ZtsClient { .setUri(ztsUrl.resolve("instance/")) .setEntity(toJsonStringEntity(payload)) .build(); - return execute(request, DefaultZtsClient::getInstanceIdentity); + return execute(request, this::getInstanceIdentity); } @Override @@ -111,7 +93,7 @@ public class DefaultZtsClient implements ZtsClient { .setUri(uri) .setEntity(toJsonStringEntity(payload)) .build(); - return execute(request, DefaultZtsClient::getInstanceIdentity); + return execute(request, this::getInstanceIdentity); } @Override @@ -189,30 +171,13 @@ public class DefaultZtsClient implements ZtsClient { }); } - private <T> T execute(HttpUriRequest request, ResponseHandler<T> responseHandler) { - try { - return client.execute(request, responseHandler); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static InstanceIdentity getInstanceIdentity(HttpResponse response) throws IOException { + private InstanceIdentity getInstanceIdentity(HttpResponse response) throws IOException { InstanceIdentityCredentials entity = readEntity(response, InstanceIdentityCredentials.class); return entity.getServiceToken() != null ? new InstanceIdentity(entity.getX509Certificate(), new NToken(entity.getServiceToken())) : new InstanceIdentity(entity.getX509Certificate()); } - private static <T> T readEntity(HttpResponse response, Class<T> entityType) throws IOException { - if (HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) { - return objectMapper.readValue(response.getEntity().getContent(), entityType); - } else { - ErrorResponseEntity errorEntity = objectMapper.readValue(response.getEntity().getContent(), ErrorResponseEntity.class); - throw new ZtsClientException(errorEntity.code, errorEntity.description); - } - } - private static URI addTrailingSlash(URI ztsUrl) { if (ztsUrl.getPath().endsWith("/")) return ztsUrl; @@ -220,34 +185,4 @@ public class DefaultZtsClient implements ZtsClient { return URI.create(ztsUrl.toString() + '/'); } - private static StringEntity toJsonStringEntity(Object entity) { - try { - return new StringEntity(objectMapper.writeValueAsString(entity), ContentType.APPLICATION_JSON); - } catch (JsonProcessingException e) { - throw new UncheckedIOException(e); - } - } - - private static CloseableHttpClient createHttpClient(Supplier<SSLContext> sslContextSupplier) { - return HttpClientBuilder.create() - .setRetryHandler(new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true)) - .setUserAgent("vespa-zts-client") - .setSSLSocketFactory(new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(sslContextSupplier), (HostnameVerifier)null)) - .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout((int)Duration.ofSeconds(10).toMillis()) - .setConnectionRequestTimeout((int)Duration.ofSeconds(10).toMillis()) - .setSocketTimeout((int)Duration.ofSeconds(20).toMillis()) - .build()) - .build(); - } - - @Override - public void close() { - try { - this.client.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityRefreshRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityRefreshRequestEntity.java index 808c1162ef1..0704fef2ae3 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityRefreshRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityRefreshRequestEntity.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.athenz.client.zts.bindings; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.yahoo.vespa.athenz.client.zts.bindings.serializers.Pkcs10CsrSerializer; +import com.yahoo.vespa.athenz.client.common.serializers.Pkcs10CsrSerializer; import com.yahoo.security.Pkcs10Csr; /** diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityResponseEntity.java index 7bd04362599..f36858ef7b8 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityResponseEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/IdentityResponseEntity.java @@ -5,8 +5,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.yahoo.vespa.athenz.client.zts.bindings.serializers.X509CertificateDeserializer; -import com.yahoo.vespa.athenz.client.zts.bindings.serializers.X509CertificateListDeserializer; +import com.yahoo.vespa.athenz.client.common.serializers.X509CertificateDeserializer; +import com.yahoo.vespa.athenz.client.common.serializers.X509CertificateListDeserializer; import java.security.cert.X509Certificate; import java.util.List; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceIdentityCredentials.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceIdentityCredentials.java index 0ab697a1c4c..b9baba85ea1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceIdentityCredentials.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceIdentityCredentials.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.yahoo.vespa.athenz.client.zts.bindings.serializers.X509CertificateDeserializer; +import com.yahoo.vespa.athenz.client.common.serializers.X509CertificateDeserializer; import java.security.cert.X509Certificate; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRefreshInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRefreshInformation.java index 0e7e94e96ac..fee91dbc15b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRefreshInformation.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRefreshInformation.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.yahoo.vespa.athenz.client.zts.bindings.serializers.Pkcs10CsrSerializer; +import com.yahoo.vespa.athenz.client.common.serializers.Pkcs10CsrSerializer; import com.yahoo.security.Pkcs10Csr; /** diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java index 1b974bcc6fc..89bfce91154 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateRequestEntity.java @@ -8,7 +8,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.yahoo.vespa.athenz.client.zts.bindings.serializers.Pkcs10CsrSerializer; +import com.yahoo.vespa.athenz.client.common.serializers.Pkcs10CsrSerializer; import com.yahoo.security.Pkcs10Csr; import java.io.IOException; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java index e80f5626843..857bfad9143 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/RoleCertificateResponseEntity.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.yahoo.vespa.athenz.client.zts.bindings.serializers.X509CertificateDeserializer; +import com.yahoo.vespa.athenz.client.common.serializers.X509CertificateDeserializer; import java.security.cert.X509Certificate; import java.time.Instant; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java index 5f44b06ab77..ff968b941a2 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java @@ -38,7 +38,7 @@ public class MockedOperationHandler implements OperationHandler { @Override public void update(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException { log.append("UPDATE: " + data.getDocumentUpdate().getId()); - log.append(data.getDocumentUpdate().getFieldUpdates().toString()); + log.append(data.getDocumentUpdate().fieldUpdates().toString()); if (data.getDocumentUpdate().getCreateIfNonExistent()) { log.append("[CREATE IF NON EXISTENT IS TRUE]"); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java index 195604b067f..c6974cff5c1 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java @@ -6,18 +6,16 @@ import com.yahoo.docproc.CallStack; import com.yahoo.docproc.DocprocService; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.Processing; -import com.yahoo.document.*; +import com.yahoo.document.DocumentOperation; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; -import com.yahoo.vdslib.Entry; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class DocprocMessageProcessor implements MessageProcessor { private final DocprocService docproc; @@ -31,7 +29,7 @@ public class DocprocMessageProcessor implements MessageProcessor { @Override public void process(Message m) { try { - List<DocumentOperation> documentBases = new ArrayList<DocumentOperation>(); + List<DocumentOperation> documentBases = new ArrayList<>(1); if (m.getType() == DocumentProtocol.MESSAGE_PUTDOCUMENT) { documentBases.add(((PutDocumentMessage) m).getDocumentPut()); @@ -49,7 +47,7 @@ public class DocprocMessageProcessor implements MessageProcessor { } } - public void processDocumentOperations(List<DocumentOperation> documentOperations, Message m) throws Exception { + private void processDocumentOperations(List<DocumentOperation> documentOperations, Message m) throws Exception { Processing processing = Processing.createProcessingFromDocumentOperations(docproc.getName(), documentOperations, new CallStack(docproc.getCallStack())); processing.setServiceName(docproc.getName()); processing.setDocprocServiceRegistry(docprocServiceRegistry); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java index 523ea0605a4..4ad5c86b663 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java @@ -24,11 +24,11 @@ public abstract class Feeder { protected final InputStream stream; protected final DocumentTypeManager docMan; - protected List<String> errors = new LinkedList<String>(); - protected boolean doAbort = true; - protected boolean createIfNonExistent = false; - protected final VespaFeedSender sender; - private final int MAX_ERRORS = 10; + protected List<String> errors = new LinkedList<>(); + private boolean doAbort = true; + private boolean createIfNonExistent = false; + private final VespaFeedSender sender; + private static final int MAX_ERRORS = 10; protected Feeder(DocumentTypeManager docMan, VespaFeedSender sender, InputStream stream) { this.docMan = docMan; @@ -44,7 +44,7 @@ public abstract class Feeder { this.createIfNonExistent = value; } - public void addException(Exception e) { + private void addException(Exception e) { String message; if (e.getMessage() != null) { message = e.getMessage().replaceAll("\"", "'"); @@ -69,7 +69,7 @@ public abstract class Feeder { protected abstract FeedReader createReader() throws Exception; public List<String> parse() { - FeedReader reader = null; + FeedReader reader; try { reader = createReader(); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SimpleFeedAccess.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SimpleFeedAccess.java index 79690d14486..98609650432 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SimpleFeedAccess.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SimpleFeedAccess.java @@ -15,5 +15,5 @@ public interface SimpleFeedAccess { void remove(DocumentId docId, TestAndSetCondition condition); void update(DocumentUpdate update, TestAndSetCondition condition); boolean isAborted(); - + void close(); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SingleSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SingleSender.java index 49f252b10f4..e0e12b26ae6 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SingleSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SingleSender.java @@ -97,12 +97,10 @@ public class SingleSender implements SimpleFeedAccess { // empty } - public void waitForPending() { - waitForPending(-1); - } - public boolean waitForPending(long timeoutMs) { return sender.waitForPending(owner, timeoutMs); } + @Override + public void close() { } } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java index d7329264bc0..b441e81a829 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java @@ -3,10 +3,6 @@ package com.yahoo.feedapi; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; -import java.util.Date; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; - /** * Wrapper class for SimpleFeedAccess to send various XML operations. */ diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java new file mode 100644 index 00000000000..3ad3e0b7f42 --- /dev/null +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java @@ -0,0 +1,82 @@ +package com.yahoo.feedhandler; + +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.feedapi.SimpleFeedAccess; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +final class ThreadedFeedAccess implements SimpleFeedAccess { + + private final SimpleFeedAccess simpleFeedAccess; + private final ExecutorService executorService; + private final Executor executor; + ThreadedFeedAccess(int numThreads, SimpleFeedAccess simpleFeedAccess) { + this.simpleFeedAccess = simpleFeedAccess; + if (numThreads <= 0) { + numThreads = Runtime.getRuntime().availableProcessors(); + } + if (numThreads > 1) { + executorService = new ThreadPoolExecutor(numThreads, numThreads, 0L, TimeUnit.SECONDS, + new SynchronousQueue<>(false), + ThreadFactoryFactory.getDaemonThreadFactory("feeder"), + new ThreadPoolExecutor.CallerRunsPolicy()); + executor = executorService; + } else { + executorService = null; + executor = new Executor() { + @Override + public void execute(Runnable command) { + command.run(); + } + }; + } + } + @Override + public void put(Document doc) { + executor.execute(() -> simpleFeedAccess.put(doc)); + } + + @Override + public void remove(DocumentId docId) { + executor.execute(() -> simpleFeedAccess.remove(docId)); + } + + @Override + public void update(DocumentUpdate update) { + executor.execute(() -> simpleFeedAccess.update(update)); + } + + @Override + public void put(Document doc, TestAndSetCondition condition) { + executor.execute(() -> simpleFeedAccess.put(doc, condition)); + } + + @Override + public void remove(DocumentId docId, TestAndSetCondition condition) { + executor.execute(() -> simpleFeedAccess.remove(docId, condition)); + } + + @Override + public void update(DocumentUpdate update, TestAndSetCondition condition) { + executor.execute(() -> simpleFeedAccess.update(update, condition)); + } + + @Override + public boolean isAborted() { + return simpleFeedAccess.isAborted(); + } + @Override + public void close() { + if (executorService != null) { + executorService.shutdown(); + } + } +} diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java index 8180bfd84ea..32c2d848b82 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java @@ -15,6 +15,7 @@ import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.Feeder; import com.yahoo.feedapi.JsonFeeder; import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.feedapi.SimpleFeedAccess; import com.yahoo.feedapi.SingleSender; import com.yahoo.feedapi.XMLFeeder; import com.yahoo.jdisc.Metric; @@ -63,10 +64,10 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { @Override public HttpResponse handle(HttpRequest request) { - return handle(request, (RouteMetricSet.ProgressCallback)null); + return handle(request, null, 1); } - public HttpResponse handle(HttpRequest request, RouteMetricSet.ProgressCallback callback) { + public HttpResponse handle(HttpRequest request, RouteMetricSet.ProgressCallback callback, int numThreads) { if (request.getProperty("status") != null) { return new MetricResponse(context.getMetrics().getMetricSet()); } @@ -85,7 +86,7 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { SingleSender sender = new SingleSender(response, getSharedSender(route), !asynchronous); sender.addMessageProcessor(properties); sender.addMessageProcessor(new DocprocMessageProcessor(getDocprocChain(request), getDocprocServiceRegistry(request))); - + ThreadedFeedAccess feedAccess = new ThreadedFeedAccess(numThreads, sender); Feeder feeder = createFeeder(sender, request); feeder.setAbortOnDocumentError(properties.getAbortOnDocumentError()); feeder.setCreateIfNonExistent(properties.getCreateIfNonExistent()); @@ -100,6 +101,7 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { } sender.done(); + feedAccess.close(); if (asynchronous) { return response; @@ -116,7 +118,7 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { } } - private Feeder createFeeder(SingleSender sender, HttpRequest request) { + private Feeder createFeeder(SimpleFeedAccess sender, HttpRequest request) { String contentType = request.getHeader("Content-Type"); if (Boolean.valueOf(request.getProperty(JSON_INPUT)) || (contentType != null && contentType.startsWith("application/json"))) { return new JsonFeeder(getDocumentTypeManager(), sender, getRequestInputStream(request)); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java index 05d0c2b81bd..7dc852bfd6d 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java @@ -36,35 +36,35 @@ public abstract class VespaFeedHandlerBase extends ThreadedHttpRequestHandler { SlobroksConfig slobroksConfig, ClusterListConfig clusterListConfig, Executor executor, - Metric metric) throws Exception { + Metric metric) { this(FeedContext.getInstance(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, metric), executor, (long)feederConfig.timeout() * 1000); } - public VespaFeedHandlerBase(FeedContext context, Executor executor) throws Exception { + VespaFeedHandlerBase(FeedContext context, Executor executor) { this(context, executor, context.getPropertyProcessor().getDefaultTimeoutMillis()); } - public VespaFeedHandlerBase(FeedContext context, Executor executor, long defaultTimeoutMillis) throws Exception { + private VespaFeedHandlerBase(FeedContext context, Executor executor, long defaultTimeoutMillis) { super(executor, context.getMetricAPI()); this.context = context; this.defaultTimeoutMillis = defaultTimeoutMillis; } - public SharedSender getSharedSender(String route) { + SharedSender getSharedSender(String route) { return context.getSharedSender(route); } - public DocprocService getDocprocChain(HttpRequest request) { + DocprocService getDocprocChain(HttpRequest request) { return context.getPropertyProcessor().getDocprocChain(request); } - public ComponentRegistry<DocprocService> getDocprocServiceRegistry(HttpRequest request) { + ComponentRegistry<DocprocService> getDocprocServiceRegistry(HttpRequest request) { return context.getPropertyProcessor().getDocprocServiceRegistry(request); } - public MessagePropertyProcessor getPropertyProcessor() { + MessagePropertyProcessor getPropertyProcessor() { return context.getPropertyProcessor(); } @@ -74,7 +74,7 @@ public abstract class VespaFeedHandlerBase extends ThreadedHttpRequestHandler { * original data stream. * @throws IllegalArgumentException if GZIP stream creation failed */ - public InputStream getRequestInputStream(HttpRequest request) { + InputStream getRequestInputStream(HttpRequest request) { if ("gzip".equals(request.getHeader("Content-Encoding"))) { try { return new GZIPInputStream(request.getData()); @@ -86,7 +86,7 @@ public abstract class VespaFeedHandlerBase extends ThreadedHttpRequestHandler { } } - protected DocumentTypeManager getDocumentTypeManager() { + DocumentTypeManager getDocumentTypeManager() { return context.getDocumentTypeManager(); } @@ -94,7 +94,7 @@ public abstract class VespaFeedHandlerBase extends ThreadedHttpRequestHandler { return context.getMetrics(); } - protected long getTimeoutMillis(HttpRequest request) { + long getTimeoutMillis(HttpRequest request) { return ParameterParser.asMilliSeconds(request.getProperty("timeout"), defaultTimeoutMillis); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java index 86512dfab73..0d23af1fec5 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java @@ -39,14 +39,15 @@ public class Arguments { } private FeederConfig.Builder feederConfigBuilder = new FeederConfig.Builder(); - private List<String> files = new ArrayList<String>(); + private List<String> files = new ArrayList<>(); private String dumpDocumentsFile = null; private String mode = "standard"; private boolean validateOnly = false; private boolean verbose = false; - SessionFactory sessionFactory = null; + SessionFactory sessionFactory; MessagePropertyProcessor propertyProcessor = null; private String priority = null; + private int numThreads = 1; public MessagePropertyProcessor getPropertyProcessor() { return propertyProcessor; @@ -83,6 +84,7 @@ public class Arguments { " feeding them.\n" + " --dumpDocuments <filename> Specify a file where documents in the put are serialized.\n" + " --priority arg Specify priority of sent messages (see documentation for priority values)\n" + + " --numthreads arg Specify how many threads to use for sending. Default is 1.\n" + " --create-if-non-existent Enable setting of create-if-non-existent to true on all document updates in the given xml feed.\n" + " -v [ --verbose ] Enable verbose output of progress.\n"); } @@ -152,6 +154,8 @@ public class Arguments { verbose = true; } else if ("--priority".equals(arg)) { priority = getParam(args, arg); + } else if ("--numthreads".equals(arg)) { + numThreads = Integer.parseInt(getParam(args, arg)); } else { files.add(arg); } @@ -183,6 +187,10 @@ public class Arguments { return priority; } + public int getNumThreads() { + return numThreads; + } + public SessionFactory getSessionFactory() { return sessionFactory; } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java index 0a926f6aae2..f80567709c4 100755 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java @@ -13,8 +13,14 @@ import com.yahoo.log.LogSetup; import com.yahoo.concurrent.SystemTimer; import com.yahoo.vespaclient.ClusterList; -import java.io.*; -import java.util.*; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.util.List; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -82,7 +88,7 @@ public class VespaFeeder { if (args.getFiles().isEmpty()) { InputStreamRequest req = new InputStreamRequest(input); setProperties(req, input); - FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output)); + FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output), args.getNumThreads()); if ( ! response.isSuccess()) { throw renderErrors(response.getErrorList()); } @@ -100,7 +106,7 @@ public class VespaFeeder { final BufferedInputStream inputSnooper = new BufferedInputStream(new FileInputStream(fileName)); setProperties(req, inputSnooper); inputSnooper.close(); - FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output)); + FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output), args.getNumThreads()); if (!response.isSuccess()) { throw renderErrors(response.getErrorList()); } diff --git a/vespaclient-java/src/main/sh/vespa-destination.sh b/vespaclient-java/src/main/sh/vespa-destination.sh index 3b11b151727..8ea2d1ca63f 100755 --- a/vespaclient-java/src/main/sh/vespa-destination.sh +++ b/vespaclient-java/src/main/sh/vespa-destination.sh @@ -75,7 +75,7 @@ exec java \ -server -enableassertions \ -XX:ThreadStackSize=512 \ -verbose:gc \ --Xms4g -Xmx4g -XX:NewRatio=1 \ +-Xms4g -Xmx4g \ -Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ -XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \ -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.dummyreceiver.DummyReceiver "$@" diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java index d1b7397de34..4de286398e9 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java @@ -68,6 +68,14 @@ public class VespaFeederTestCase { } @Test + public void requireThatnumThreadsBeParsed() throws Exception { + String argsS="--numthreads 5"; + Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply()); + assertEquals(5, arguments.getNumThreads()); + assertEquals(1, new Arguments("".split(" "), DummySessionFactory.createWithAutoReply()).getNumThreads()); + } + + @Test public void testHelp() throws Exception { String argsS="-h"; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/RestrictedServerCnxnFactory.java b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/RestrictedServerCnxnFactory.java index d7f42c7e6e9..dab9ddb243b 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/RestrictedServerCnxnFactory.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/RestrictedServerCnxnFactory.java @@ -16,7 +16,8 @@ import java.util.Set; import java.util.logging.Logger; /** - * This class is created by zookeeper by reflection, see the ZooKeeperServer constructor. + * This class is created by zookeeper by reflection, see the ZooKeeperServer constructor. It will only work + * when using ZooKeeper 3.4 * * @author bratseth */ @@ -66,9 +67,8 @@ public class RestrictedServerCnxnFactory extends NIOServerCnxnFactory { String environmentAllowedZooKeeperClients = System.getenv("vespa_zkfacade__restrict"); if (environmentAllowedZooKeeperClients != null) return ImmutableSet.copyOf(toHostnameSet(environmentAllowedZooKeeperClients)); - - // No environment setting -> use static field - return ZooKeeperServer.getAllowedClientHostnames(); + else + return ImmutableSet.of(); } private Set<String> toHostnameSet(String hostnamesString) { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java index 405afcd3c39..9c580b4f9ce 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; -import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.log.LogLevel; @@ -23,29 +21,18 @@ import java.util.stream.Collectors; */ public class ZooKeeperServer extends AbstractComponent implements Runnable { - /** - * The set of hosts which can access the ZooKeeper server in this VM, or empty - * to allow access from anywhere. - * This belongs logically to the server instance and is final, but must be static to make it accessible - * from RestrictedServerCnxnFactory, which is created by ZK through reflection. - */ - private static ImmutableSet<String> allowedClientHostnames = ImmutableSet.of(); - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ZooKeeperServer.class.getName()); private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable"; static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer"; private final Thread zkServerThread; private final ZookeeperServerConfig zookeeperServerConfig; - ZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig, ConfigserverConfig configserverConfig, boolean startServer) { + ZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig, boolean startServer) { this.zookeeperServerConfig = zookeeperServerConfig; System.setProperty("zookeeper.jmx.log4j.disable", "true"); System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, "" + zookeeperServerConfig.juteMaxBuffer()); System.setProperty("zookeeper.serverCnxnFactory", "com.yahoo.vespa.zookeeper.RestrictedServerCnxnFactory"); - if (configserverConfig.hostedVespa()) // restrict access to config servers only - allowedClientHostnames = ImmutableSet.copyOf(zookeeperServerHostnames(zookeeperServerConfig)); - writeConfigToDisk(zookeeperServerConfig); zkServerThread = new Thread(this, "zookeeper server"); if (startServer) { @@ -54,13 +41,10 @@ public class ZooKeeperServer extends AbstractComponent implements Runnable { } @Inject - public ZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig, ConfigserverConfig configserverConfig) { - this(zookeeperServerConfig, configserverConfig, true); + public ZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig) { + this(zookeeperServerConfig, true); } - /** Returns the hosts which are allowed to access this ZooKeeper server, or empty to allow access from anywhere */ - public static ImmutableSet<String> getAllowedClientHostnames() { return allowedClientHostnames; } - private void writeConfigToDisk(ZookeeperServerConfig config) { String configFilePath = getDefaults().underVespaHome(config.zooKeeperConfigFile()); new File(configFilePath).getParentFile().mkdirs(); @@ -83,6 +67,10 @@ public class ZooKeeperServer extends AbstractComponent implements Runnable { sb.append("clientPort=").append(config.clientPort()).append("\n"); sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n"); sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n"); + // See http://zookeeper.apache.org/doc/r3.4.13/zookeeperAdmin.html#sc_zkCommands + // Includes all available commands in 3.4, except 'wchc' and 'wchp' + // Mandatory when using ZooKeeper 3.5 + sb.append("4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n"); if (config.server().size() > 1) { ensureThisServerIsRepresented(config.myid(), config.server()); for (ZookeeperServerConfig.Server server : config.server()) { diff --git a/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java b/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java index 626e5bf0627..db1852d9d2a 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.io.IOUtils; import org.junit.Rule; @@ -54,7 +53,7 @@ public class ZooKeeperServerTest { } private void createServer(ZookeeperServerConfig.Builder builder) { - new ZooKeeperServer(new ZookeeperServerConfig(builder), new ConfigserverConfig(new ConfigserverConfig.Builder()), false); + new ZooKeeperServer(new ZookeeperServerConfig(builder), false); } @Test(expected = RuntimeException.class) @@ -110,7 +109,8 @@ public class ZooKeeperServerTest { "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" + "clientPort=2181\n" + "autopurge.purgeInterval=1\n" + - "autopurge.snapRetainCount=15\n"; + "autopurge.snapRetainCount=15\n" + + "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n"; validateConfigFile(cfgFile, expected); } @@ -125,6 +125,7 @@ public class ZooKeeperServerTest { "clientPort=2181\n" + "autopurge.purgeInterval=1\n" + "autopurge.snapRetainCount=15\n" + + "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + "server.1=foo:321:123\n" + "server.2=bar:432:234\n" + "server.3=baz:543:345\n"; |