diff options
56 files changed, 692 insertions, 1156 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java index ca8eadd8d1f..158fbfb175f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java @@ -13,7 +13,7 @@ import java.util.stream.Collectors; * endpoint (endpointId) and the name of the container cluster that the endpoint * should point to. * - * If the endpointId is not set, it will default to the same as the containerId. + * If the endpoint is not set it will default to the string "default". * * @author ogronnesby */ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 650f68591b6..72a806bb7be 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -161,7 +162,7 @@ public class DeploymentSpecXmlReader { final var endpointsElement = XML.getChild(root, endpointsTag); if (endpointsElement == null) { return Collections.emptyList(); } - final var endpoints = new ArrayList<Endpoint>(); + final var endpoints = new LinkedHashMap<String, Endpoint>(); for (var endpointElement : XML.getChildren(endpointsElement, endpointTag)) { final Optional<String> rotationId = stringAttribute("id", endpointElement); @@ -184,13 +185,13 @@ public class DeploymentSpecXmlReader { } var endpoint = new Endpoint(rotationId, containerId.get(), regions); - if (endpoints.contains(endpoint)) { - throw new IllegalArgumentException("Duplicate 'endpoint' in 'endpoints' tag"); + if (endpoints.containsKey(endpoint.endpointId())) { + throw new IllegalArgumentException("Duplicate attribute 'id' on 'endpoint': " + endpoint.endpointId()); } - endpoints.add(endpoint); + endpoints.put(endpoint.endpointId(), endpoint); } - return endpoints; + return List.copyOf(endpoints.values()); } /** diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java deleted file mode 100644 index 87970458855..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author mortent - */ -public class AttributeMapping { - - private final String attribute; - private final List<String> chefPath; - - private AttributeMapping(String attribute, List<String> chefPath) { - this.chefPath = chefPath; - this.attribute = attribute; - } - - public static AttributeMapping simpleMapping(String attribute) { - return new AttributeMapping(attribute, Collections.singletonList(attribute)); - } - - public static AttributeMapping deepMapping(String attribute, List<String> chefPath) { - return new AttributeMapping(attribute, chefPath); - } - - public String toString() { - return String.format("\"%s\": [%s]", attribute, - chefPath.stream().map(s -> String.format("\"%s\"", s)) - .collect(Collectors.joining(",")) - ); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java deleted file mode 100644 index 693947b6f61..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef; - - -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefEnvironment; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefResource; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.Client; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.CookBook; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.NodeResult; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; - -import java.net.URL; -import java.util.List; - -public interface Chef { - - ChefResource getApi(); - - ChefNode getNode(String name); - - Client getClient(String name); - - ChefNode deleteNode(String name); - - Client deleteClient(String name); - - NodeResult searchNodeByFQDN(String fqdn); - - NodeResult searchNodes(String query); - - PartialNodeResult partialSearchNodes(String query, List<AttributeMapping> attributeMappings); - - void copyChefEnvironment(String fromEnvironmentName, String toEnvironmentName); - - ChefEnvironment getChefEnvironment(String environmentName); - - CookBook getCookbook(String cookbookName, String cookbookVersion); - - String downloadResource(URL resourceURL); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java deleted file mode 100644 index bd19cfe6ce1..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef; - -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefEnvironment; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefResource; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.Client; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.CookBook; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.NodeResult; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; - -import javax.ws.rs.NotFoundException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author mpolden - */ -public class ChefMock implements Chef { - - private final NodeResult result; - private final PartialNodeResult partialResult; - private final List<String> chefEnvironments; - - public ChefMock() { - result = new NodeResult(); - result.rows = new ArrayList<>(); - partialResult = new PartialNodeResult(); - partialResult.rows = new ArrayList<>(); - chefEnvironments = new ArrayList<>(); - chefEnvironments.add("hosted-verified-prod"); - chefEnvironments.add("hosted-infra-cd"); - } - - @Override - public ChefResource getApi() { - return null; - } - - @Override - public ChefNode getNode(String name) { - return null; - } - - @Override - public Client getClient(String name) { - return null; - } - - @Override - public ChefNode deleteNode(String name) { - return null; - } - - @Override - public Client deleteClient(String name) { - return null; - } - - public ChefMock addSearchResult(ChefNode node) { - result.rows.add(node); - return this; - } - - public ChefMock addPartialResult(List<PartialNode> partialNodes) { - partialResult.rows.addAll(partialNodes); - return this; - } - - @Override - public NodeResult searchNodeByFQDN(String fqdn) { - return result; - } - - @Override - public NodeResult searchNodes(String query) { - return result; - } - - @Override - public PartialNodeResult partialSearchNodes(String query, List<AttributeMapping> returnAttributes) { - PartialNodeResult partialNodeResult = new PartialNodeResult(); - partialNodeResult.rows = new ArrayList<>(); - partialNodeResult.rows.addAll(partialResult.rows); - result.rows.stream() - .map(chefNode -> { - Map<String, String> data = new HashMap<>(); - data.put("fqdn", chefNode.name); - return new PartialNode(data); - }) - .forEach(node -> partialNodeResult.rows.add(node)); - return partialNodeResult; - } - - @Override - public void copyChefEnvironment(String fromEnvironmentName, String toEnvironmentName) { - if(!chefEnvironments.contains(fromEnvironmentName)) { - throw new NotFoundException(String.format("Source chef environment %s does not exist", fromEnvironmentName)); - } - chefEnvironments.add(toEnvironmentName); - } - - @Override - public ChefEnvironment getChefEnvironment(String environmentName) { - return null; - } - - @Override - public CookBook getCookbook(String cookbookName, String cookbookVersion) { - return null; - } - - @Override - public String downloadResource(URL resourceURL) { - return ""; - } -} - diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java deleted file mode 100644 index 5d3d4b87b74..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.chef; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java deleted file mode 100644 index 8576949280b..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Map; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class ChefEnvironment { - - @JsonProperty("name") - private String name; - - @JsonProperty("default_attributes") - private Map<String, Object> attributes; - @JsonProperty("override_attributes") - private Map<String, Object> overrideAttributes; - @JsonProperty("description") - private String description; - @JsonProperty("cookbook_versions") - private Map<String, String> cookbookVersions; - - // internal - @JsonProperty("json_class") - private final String _jsonClass = "Chef::Environment"; - @JsonProperty("chef_type") - private final String _chefType = "environment"; - - public static Builder builder() { - return new Builder(); - } - - public String getName() { - return name; - } - - public Builder copy() { - return builder() - .name(name) - .attributes(attributes) - .overrideAttributes(overrideAttributes) - .cookbookVersions(cookbookVersions) - .description(description); - } - - public String getDescription() { - return description; - } - - public Map<String, String> getCookbookVersions() { - return cookbookVersions; - } - - public Map<String, Object> getAttributes() { - return attributes; - } - - public Map<String, Object> getOverrideAttributes() { - return overrideAttributes; - } - - public static class Builder { - private String name; - private Map<String, Object> attributes; - private String description; - private Map<String, Object> overrideAttributes; - private Map<String, String> cookbookVersions; - - public Builder name(String name){ - this.name = name; - return this; - } - - public Builder attributes(Map<String, Object> defaultAttributes) { - this.attributes = defaultAttributes; - return this; - } - - public Builder overrideAttributes(Map<String, Object> overrideAttributes) { - this.overrideAttributes = overrideAttributes; - return this; - } - - public Builder cookbookVersions(Map<String, String> cookbookVersions) { - this.cookbookVersions = cookbookVersions; - return this; - } - - public Builder description(String description) { - this.description = description; - return this; - } - - public ChefEnvironment build() { - ChefEnvironment chefEnvironment = new ChefEnvironment(); - chefEnvironment.name = name; - chefEnvironment.description = description; - chefEnvironment.cookbookVersions = cookbookVersions; - chefEnvironment.attributes = attributes; - chefEnvironment.overrideAttributes = overrideAttributes; - - return chefEnvironment; - } - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java deleted file mode 100644 index 08d9a1045e8..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class ChefNode { - - @JsonProperty("name") - public String name; - - @JsonProperty("chef_environment") - public String chefEnvironment; - - @JsonProperty("run_list") - public List<String> runList; - - @JsonProperty("json_class") - public String jsonClass; - - @JsonProperty("chef_type") - public String chefType; - - @JsonProperty("automatic") - public Map<String, Object> automaticAttributes; - - @JsonProperty("normal") - public Map<String, Object> normalAttributes; - - @JsonProperty("default") - public Map<String, Object> defaultAttributes; - - @JsonProperty("override") - public Map<String, Object> overrideAttributes; - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(ChefNode src) { - return new Builder(src); - } - - public static class Builder { - private String name; - private String chefEnvironment; - private List<String> runList; - private String jsonClass; - private String chefType; - private Map<String, Object> automaticAttributes; - private Map<String, Object> normalAttributes; - private Map<String, Object> defaultAttributes; - private Map<String, Object> overrideAttributes; - - private Builder(){} - - private Builder(ChefNode src){ - this.name = src.name; - this.chefEnvironment = src.chefEnvironment; - this.runList = new ArrayList<>(src.runList); - this.jsonClass = src.jsonClass; - this.chefType = src.chefType; - this.automaticAttributes = new HashMap<>(src.automaticAttributes); - this.normalAttributes = new HashMap<>(src.normalAttributes); - this.defaultAttributes = new HashMap<>(src.defaultAttributes); - this.overrideAttributes = new HashMap<>(src.overrideAttributes); - } - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder chefEnvironment(String chefEnvironment) { - this.chefEnvironment = chefEnvironment; - return this; - } - - public ChefNode build(){ - ChefNode node = new ChefNode(); - node.name = this.name; - node.chefEnvironment = this.chefEnvironment; - node.runList = this.runList; - node.jsonClass = this.jsonClass; - node.chefType = this.chefType; - node.automaticAttributes = this.automaticAttributes; - node.overrideAttributes = this.overrideAttributes; - node.defaultAttributes = this.defaultAttributes; - node.normalAttributes = this.normalAttributes; - return node; - } - - } - - @Override - public String toString() { - return "Node{" + - "name='" + name + '\'' + - ", chefEnvironment='" + chefEnvironment + '\'' + - ", runList=" + runList + - ", jsonClass='" + jsonClass + '\'' + - ", chefType='" + chefType + '\'' + - ", automaticAttributes=" + automaticAttributes + - ", normalAttributes=" + normalAttributes + - ", defaultAttributes=" + defaultAttributes + - ", overrideAttributes=" + overrideAttributes + - '}'; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java deleted file mode 100644 index 98eeb0770fc..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; -import java.util.List; - -/** - * @author mortent - * @author mpolden - */ - -@Path("/") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public interface ChefResource { - - @Path("/organizations/{organization}/environments/{environment}/nodes") - @Consumes("application/json") - @GET - List<String> getNodes(@PathParam("organization") String organization, @PathParam("environment") String environment); - - @GET - @Path("/organizations/{organization}/nodes/{nodename}") - ChefNode getNode(@PathParam("organization") String organization, @PathParam("nodename") String nodename); - - @PUT - @Path("/organizations/{organization}/nodes/{nodename}") - ChefNode updateNode(@PathParam("organization") String organization, @PathParam("nodename") String nodeName, String node); - - @DELETE - @Path("/organizations/{organization}/nodes/{nodename}") - ChefNode deleteNode(@PathParam("organization") String organization, @PathParam("nodename") String nodeName); - - @GET - @Path("/organizations/{organization}/clients/{name}") - Client getClient(@PathParam("organization") String organization, @PathParam("name") String name); - - @DELETE - @Path("/organizations/{organization}/clients/{name}") - Client deleteClient(@PathParam("organization") String organization, @PathParam("name") String name); - - @GET - @Path("/organizations/{organization}/environments/{environment}") - ChefEnvironment getEnvironment(@PathParam("organization") String organization, @PathParam("environment") String environment); - - @PUT - @Path("/organizations/{organization}/environments/{name}") - String updateEnvironment(@PathParam("organization") String organization, @PathParam("name") String chefEnvironmentName, String contentAsString); - - @POST - @Path("/organizations/{organization}/environments") - String createEnvironment(@PathParam("organization") String organization, String contentAsString); - - @GET - @Path("/organizations/{organization}/search/node") - NodeResult searchNode(@PathParam("organization") String organization, @QueryParam("q") String query); - - @POST - @Path("/organizations/{organization}/search/node") - PartialNodeResult partialSearchNode(@PathParam("organization") String organization, @QueryParam("q") String query, @QueryParam("rows") int rows, String keys); - - @GET - @Path("/organizations/{organization}/cookbooks/{name}/{version}") - CookBook getCookBook(@PathParam("organization") String organization, @PathParam("name") String name, @PathParam("version") String version); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java deleted file mode 100644 index 0ea9b0e9997..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author mpolden - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class Client { - - @JsonProperty("name") - public String name; - @JsonProperty("validator") - public boolean validator; - - @Override - public String toString() { - return "Client{" + - "name='" + name + '\'' + - ", validator=" + validator + - '}'; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java deleted file mode 100644 index ab49ac9ff60..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class CookBook { - public final String name; - public final List<Attributes> attributes; - - public CookBook(@JsonProperty("name") String name, @JsonProperty("attributes") List<Attributes> attributes) { - this.name = name; - this.attributes = attributes; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Attributes { - public final String name; - public final String url; - - public Attributes(@JsonProperty("name") String name, @JsonProperty("url") String url) { - this.name = name; - this.url = url; - } - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java deleted file mode 100644 index e3ab431473f..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * @author mpolden - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class NodeResult { - @JsonProperty("total") - public int total; - @JsonProperty("start") - public int start; - @JsonProperty("rows") - public List<ChefNode> rows; -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java deleted file mode 100644 index f4aa90021b1..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Map; -import java.util.Optional; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class PartialNode { - - @JsonProperty("data") - private final Map<String, String> data; - - @JsonCreator - public PartialNode(@JsonProperty("data") Map<String, String> data) { - this.data = data; - } - - public Optional<String> getValue(String key) { - return Optional.ofNullable(data.get(key)); - } - - public String getFqdn() { - return getValue("fqdn").orElse(""); - } - - public String getName() { - return getValue("name").orElse(""); - } - - public Double getOhaiTime() { - return Double.parseDouble(getValue("ohai_time").orElse("0.0")); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java deleted file mode 100644 index 9925237a193..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class PartialNodeResult { - @JsonProperty("total") - public int total; - @JsonProperty("start") - public int start; - @JsonProperty("rows") - public List<PartialNode> rows; -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java deleted file mode 100644 index 7d06571507e..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index ba00203ec34..9eae2965c45 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -30,7 +30,7 @@ public interface ConfigServer { } PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationNames, - List<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content); + Set<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content); void restart(DeploymentId deployment, Optional<Hostname> hostname); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 6745b75a9ea..1ec75e0c998 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -141,9 +141,7 @@ enum PathGroup { Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/jobreport", - "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/jobreport", - "/application/v4/tenant/{tenant}/application/{application}/promote", - "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/jobreport"), /** Paths which contain (not very strictly) classified information about customers. */ classifiedTenantInfo(Optional.of("/api"), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 9ca73d27120..d8b56502fc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -218,12 +219,18 @@ public class Application { return rotations; } + /** Returns the default global endpoints for this in given system - for a given endpoint ID */ + public EndpointList endpointsIn(SystemName system, EndpointId endpointId) { + if (rotations.isEmpty()) return EndpointList.EMPTY; + return EndpointList.create(id, endpointId, system); + } + /** Returns the default global endpoints for this in given system */ public EndpointList endpointsIn(SystemName system) { - // TODO: Do we need to change something here? .defaultGlobalId seems like it is - // TODO: making some assumptions on naming. if (rotations.isEmpty()) return EndpointList.EMPTY; - return EndpointList.defaultGlobal(id, system); + final var endpointStream = rotations.stream() + .flatMap(rotation -> EndpointList.create(id, rotation.endpointId(), system).asList().stream()); + return EndpointList.of(endpointStream); } public Optional<String> pemDeployKey() { return pemDeployKey; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 197dda8c409..d015e65a5e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableList; +import com.yahoo.collections.ArraySet; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -9,6 +10,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -30,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.Applicatio import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; @@ -87,6 +90,7 @@ import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -97,6 +101,7 @@ import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; @@ -126,6 +131,7 @@ public class ApplicationController { private final RoutingPolicies routingPolicies; private final Clock clock; private final BooleanFlag redirectLegacyDnsFlag; + private final BooleanFlag useMultipleEndpoints; private final DeploymentTrigger deploymentTrigger; private final BooleanFlag provisionApplicationCertificate; private final ApplicationCertificateProvider applicationCertificateProvider; @@ -143,6 +149,7 @@ public class ApplicationController { this.routingPolicies = new RoutingPolicies(controller); this.clock = clock; this.redirectLegacyDnsFlag = Flags.REDIRECT_LEGACY_DNS_NAMES.bindTo(controller.flagSource()); + this.useMultipleEndpoints = Flags.MULTIPLE_GLOBAL_ENDPOINTS.bindTo(controller.flagSource()); this.artifactRepository = artifactRepository; this.applicationStore = applicationStore; @@ -293,7 +300,8 @@ public class ApplicationController { Version platformVersion; ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; - Set<String> rotationNames = new HashSet<>(); + Set<String> legacyRotations = new LinkedHashSet<>(); + Set<ContainerEndpoint> endpoints = new LinkedHashSet<>(); ApplicationCertificate applicationCertificate; try (Lock lock = lock(applicationId)) { @@ -332,13 +340,34 @@ public class ApplicationController { // TODO: Remove this when all packages are validated upon submission, as in ApplicationApiHandler.submit(...). verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity); + // Assign global rotation - application = withRotation(application, zone); - Application app = application.get(); - // Include global DNS names - app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(rotationNames::add); - // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header - app.rotations().stream().map(RotationId::asString).forEach(rotationNames::add); + if (useMultipleEndpoints.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()).value()) { + application = withRotation(application, zone); + + // Include global DNS names + Application app = application.get(); + app.assignedRotations().stream() + .filter(assignedRotation -> assignedRotation.regions().contains(zone.region())) + .map(assignedRotation -> { + return new ContainerEndpoint( + assignedRotation.clusterId().value(), + Stream.concat( + app.endpointsIn(controller.system(), assignedRotation.endpointId()).legacy(false).asList().stream().map(Endpoint::dnsName), + app.rotations().stream().map(RotationId::asString) + ).collect(Collectors.toList()) + ); + }) + .forEach(endpoints::add); + } else { + application = withRotationLegacy(application, zone); + + // Add both the names we have in DNS for each endpoint as well as name of the rotation so healthchecks works + Application app = application.get(); + app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(legacyRotations::add); + app.rotations().stream().map(RotationId::asString).forEach(legacyRotations::add); + } + // Get application certificate (provisions a new certificate if missing) application = withApplicationCertificate(application); @@ -354,7 +383,7 @@ public class ApplicationController { // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); - ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames, applicationCertificate); + ActivateResult result = deploy(applicationId, applicationPackage, zone, options, legacyRotations, endpoints, applicationCertificate); lockOrThrow(applicationId, application -> store(application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), @@ -421,7 +450,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Set.of(), /* No application cert */ null); + return deploy(application.id(), applicationPackage, zone, options, Set.of(), Set.of(), /* No application cert */ null); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -429,16 +458,16 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { - return deploy(tester.id(), applicationPackage, zone, options, Set.of(), /* No application cert for tester*/ null); + return deploy(tester.id(), applicationPackage, zone, options, Set.of(), Set.of(), /* No application cert for tester*/ null); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, - Set<String> rotationNames, ApplicationCertificate applicationCertificate) { + Set<String> legacyRotations, Set<ContainerEndpoint> endpoints, ApplicationCertificate applicationCertificate) { DeploymentId deploymentId = new DeploymentId(application, zone); try { ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationCertificate, applicationPackage.zippedContent()); + configServer.deploy(deploymentId, deployOptions, legacyRotations, endpoints, applicationCertificate, applicationPackage.zippedContent()); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { @@ -449,19 +478,20 @@ public class ApplicationController { } /** Makes sure the application has a global rotation, if eligible. */ - private LockedApplication withRotation(LockedApplication application, ZoneId zone) { + private LockedApplication withRotationLegacy(LockedApplication application, ZoneId zone) { if (zone.environment() == Environment.prod && application.get().deploymentSpec().globalServiceId().isPresent()) { try (RotationLock rotationLock = rotationRepository.lock()) { Rotation rotation = rotationRepository.getOrAssignRotation(application.get(), rotationLock); - application = application.with(List.of(new AssignedRotation(new ClusterSpec.Id(application.get().deploymentSpec().globalServiceId().get()), EndpointId.default_(), rotation.id()))); + application = application.with(createDefaultGlobalIdRotation(application.get(), rotation)); store(application); // store assigned rotation even if deployment fails boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) - .value(); + .value(); EndpointList globalEndpoints = application.get() - .endpointsIn(controller.system()) - .scope(Endpoint.Scope.global); + .endpointsIn(controller.system()) + .scope(Endpoint.Scope.global); + globalEndpoints.main().ifPresent(mainEndpoint -> { registerCname(mainEndpoint.dnsName(), rotation.name()); if (redirectLegacyDns) { @@ -475,6 +505,54 @@ public class ApplicationController { return application; } + private List<AssignedRotation> createDefaultGlobalIdRotation(Application application, Rotation rotation) { + // This is guaranteed by .withRotationLegacy, but add this to make inspections accept the use of .get() below + assert application.deploymentSpec().globalServiceId().isPresent(); + + final Set<RegionName> regions = application.deploymentSpec().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + + final var assignment = new AssignedRotation( + ClusterSpec.Id.from(application.deploymentSpec().globalServiceId().get()), + EndpointId.default_(), + rotation.id(), + regions + ); + + return List.of(assignment); + } + + /** Makes sure the application has a global rotation, if eligible. */ + private LockedApplication withRotation(LockedApplication application, ZoneId zone) { + if (zone.environment() == Environment.prod) { + try (RotationLock rotationLock = rotationRepository.lock()) { + final var rotations = rotationRepository.getOrAssignRotations(application.get(), rotationLock); + application = application.with(rotations); + store(application); // store assigned rotation even if deployment fails + registerAssignedRotationCnames(application.get()); + } + } + return application; + } + + private void registerAssignedRotationCnames(Application application) { + application.assignedRotations().forEach(assignedRotation -> { + final var endpoints = application + .endpointsIn(controller.system(), assignedRotation.endpointId()) + .scope(Endpoint.Scope.global); + + final var maybeRotation = rotationRepository.getRotation(assignedRotation.rotationId()); + + maybeRotation.ifPresent(rotation -> { + endpoints.main().ifPresent(mainEndpoint -> { + registerCname(mainEndpoint.dnsName(), rotation.name()); + }); + }); + }); + } + private LockedApplication withApplicationCertificate(LockedApplication application) { ApplicationId applicationId = application.get().id(); @@ -631,8 +709,14 @@ public class ApplicationController { applicationStore.removeAll(id); applicationStore.removeAll(TesterId.of(id)); - EndpointList endpoints = application.get().endpointsIn(controller.system()); - endpoints.asList().stream().map(Endpoint::dnsName).forEach(name -> controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal)); + application.get().assignedRotations().forEach(assignedRotation -> { + final var endpoints = application.get().endpointsIn(controller.system(), assignedRotation.endpointId()); + endpoints.asList().stream() + .map(Endpoint::dnsName) + .forEach(name -> { + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal); + }); + }); log.info("Deleted " + application); })); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index ed81d08c533..08c95d1ecab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; @@ -79,7 +78,6 @@ public class Controller extends AbstractComponent { private final ZoneRegistry zoneRegistry; private final ConfigServer configServer; private final MetricsService metricsService; - private final Chef chef; private final Mailer mailer; private final AuditLogger auditLogger; private final FlagSource flagSource; @@ -95,13 +93,13 @@ public class Controller extends AbstractComponent { @Inject public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, ZoneRegistry zoneRegistry, ConfigServer configServer, MetricsService metricsService, - RoutingGenerator routingGenerator, Chef chef, + RoutingGenerator routingGenerator, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider) { this(curator, rotationsConfig, gitHub, zoneRegistry, - configServer, metricsService, routingGenerator, chef, + configServer, metricsService, routingGenerator, Clock.systemUTC(), accessControl, artifactRepository, applicationStore, testerCloud, buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource, mavenRepository, applicationCertificateProvider); @@ -110,7 +108,7 @@ public class Controller extends AbstractComponent { public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, ZoneRegistry zoneRegistry, ConfigServer configServer, MetricsService metricsService, - RoutingGenerator routingGenerator, Chef chef, Clock clock, + RoutingGenerator routingGenerator, Clock clock, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Supplier<String> hostnameSupplier, @@ -122,7 +120,6 @@ public class Controller extends AbstractComponent { this.zoneRegistry = Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null"); this.configServer = Objects.requireNonNull(configServer, "ConfigServer cannot be null"); this.metricsService = Objects.requireNonNull(metricsService, "MetricsService cannot be null"); - this.chef = Objects.requireNonNull(chef, "Chef cannot be null"); this.clock = Objects.requireNonNull(clock, "Clock cannot be null"); this.mailer = Objects.requireNonNull(mailer, "Mailer cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); @@ -294,10 +291,6 @@ public class Controller extends AbstractComponent { return zoneRegistry.system(); } - public Chef chefClient() { - return chef; - } - public CuratorDb curator() { return curator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index e1ed278a79e..ec13066d069 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -1,12 +1,16 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.rotation.RotationId; +import java.util.Collection; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** - * Contains the tuple of [clusterId, endpointId, rotationId], to keep track + * Contains the tuple of [clusterId, endpointId, rotationId, regions[]], to keep track * of which services have assigned which rotations under which name. * * @author ogronnesby @@ -15,16 +19,19 @@ public class AssignedRotation { private final ClusterSpec.Id clusterId; private final EndpointId endpointId; private final RotationId rotationId; + private final Set<RegionName> regions; - public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId) { + public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId, Set<RegionName> regions) { this.clusterId = requireNonEmpty(clusterId, clusterId.value(), "clusterId"); this.endpointId = Objects.requireNonNull(endpointId); this.rotationId = Objects.requireNonNull(rotationId); + this.regions = Set.copyOf(Objects.requireNonNull(regions)); } public ClusterSpec.Id clusterId() { return clusterId; } public EndpointId endpointId() { return endpointId; } public RotationId rotationId() { return rotationId; } + public Set<RegionName> regions() { return regions; } @Override public String toString() { @@ -32,6 +39,7 @@ public class AssignedRotation { "clusterId=" + clusterId + ", endpointId='" + endpointId + '\'' + ", rotationId=" + rotationId + + ", regions=" + regions + '}'; } @@ -42,12 +50,13 @@ public class AssignedRotation { AssignedRotation that = (AssignedRotation) o; return clusterId.equals(that.clusterId) && endpointId.equals(that.endpointId) && - rotationId.equals(that.rotationId); + rotationId.equals(that.rotationId) && + regions.equals(that.regions); } @Override public int hashCode() { - return Objects.hash(clusterId, endpointId, rotationId); + return Objects.hash(clusterId, endpointId, rotationId, regions); } private static <T> T requireNonEmpty(T object, String value, String field) { @@ -58,4 +67,14 @@ public class AssignedRotation { } return object; } + + /** Convenience method intended for tests */ + public static AssignedRotation fromStrings(String clusterId, String endpointId, String rotationId, Collection<String> regions) { + return new AssignedRotation( + new ClusterSpec.Id(clusterId), + new EndpointId(endpointId), + new RotationId(rotationId), + regions.stream().map(RegionName::from).collect(Collectors.toSet()) + ); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 5026ca75a83..5dccd5c8120 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -217,6 +217,7 @@ public class Endpoint { private ZoneId zone; private ClusterSpec.Id cluster; private RotationName rotation; + private EndpointId endpointId; private Port port; private boolean legacy = false; private boolean directRouting = false; @@ -227,8 +228,8 @@ public class Endpoint { /** Sets the cluster and zone target of this */ public EndpointBuilder target(ClusterSpec.Id cluster, ZoneId zone) { - if (rotation != null) { - throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + if (rotation != null || endpointId != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); } this.cluster = cluster; this.zone = zone; @@ -237,13 +238,22 @@ public class Endpoint { /** Sets the rotation target of this */ public EndpointBuilder target(RotationName rotation) { - if (cluster != null && zone != null) { - throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + if ((cluster != null && zone != null) || endpointId != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); } this.rotation = rotation; return this; } + /** Sets the endpoint ID as defines in deployments.xml */ + public EndpointBuilder named(EndpointId endpointId) { + if (rotation != null || cluster != null || zone != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); + } + this.endpointId = endpointId; + return this; + } + /** Sets the port of this */ public EndpointBuilder on(Port port) { this.port = port; @@ -269,6 +279,8 @@ public class Endpoint { name = cluster.value(); } else if (rotation != null) { name = rotation.value(); + } else if (endpointId != null) { + name = endpointId.id(); } else { throw new IllegalArgumentException("Must set either cluster or rotation target"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index 0c04a1f099c..feedc1c8f9d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -67,16 +67,14 @@ public class EndpointList { } /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ - public static EndpointList defaultGlobal(ApplicationId application, SystemName system) { - // Rotation name is always default in the routing layer - RotationName rotation = RotationName.from("default"); + public static EndpointList create(ApplicationId application, EndpointId endpointId, SystemName system) { switch (system) { case cd: case main: return new EndpointList(List.of( - Endpoint.of(application).target(rotation).on(Port.plain(4080)).legacy().in(system), - Endpoint.of(application).target(rotation).on(Port.tls(4443)).legacy().in(system), - Endpoint.of(application).target(rotation).on(Port.tls(4443)).in(system) + Endpoint.of(application).named(endpointId).on(Port.plain(4080)).legacy().in(system), + Endpoint.of(application).named(endpointId).on(Port.tls(4443)).legacy().in(system), + Endpoint.of(application).named(endpointId).on(Port.tls(4443)).in(system) )); } return EMPTY; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index e840deb062c..f34c24c497a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -5,15 +5,14 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; -import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; +import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; @@ -59,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent { @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, - JobControl jobControl, Metric metric, Chef chefClient, + JobControl jobControl, Metric metric, DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient, ContactRetriever contactRetriever, @@ -71,7 +70,7 @@ public class ControllerMaintenance extends AbstractComponent { this.jobControl = jobControl; deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl); deploymentIssueReporter = new DeploymentIssueReporter(controller, deploymentIssues, maintenanceInterval, jobControl); - metricsReporter = new MetricsReporter(controller, metric, chefClient, jobControl, controller.system()); + metricsReporter = new MetricsReporter(controller, metric, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(controller, Duration.ofMinutes(1), jobControl); versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(1), jobControl); upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 5e3f21c6b98..c7b76696d84 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -3,14 +3,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -24,10 +19,8 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; /** @@ -36,7 +29,6 @@ import java.util.stream.Collectors; */ public class MetricsReporter extends Maintainer { - public static final String CONVERGENCE_METRIC = "seconds.since.last.chef.convergence"; public static final String DEPLOYMENT_FAIL_METRIC = "deployment.failurePercentage"; public static final String DEPLOYMENT_AVERAGE_DURATION = "deployment.averageDuration"; public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; @@ -46,27 +38,16 @@ public class MetricsReporter extends Maintainer { public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; private final Metric metric; - private final Chef chefClient; private final Clock clock; - private final SystemName system; - public MetricsReporter(Controller controller, Metric metric, Chef chefClient, JobControl jobControl, - SystemName system) { - this(controller, metric, chefClient, Clock.systemUTC(), jobControl, system); - } - - public MetricsReporter(Controller controller, Metric metric, Chef chefClient, Clock clock, - JobControl jobControl, SystemName system) { + public MetricsReporter(Controller controller, Metric metric, JobControl jobControl) { super(controller, Duration.ofMinutes(1), jobControl); // use fixed rate for metrics this.metric = metric; - this.chefClient = chefClient; - this.clock = clock; - this.system = system; + this.clock = controller.clock(); } @Override public void maintain() { - reportChefMetrics(); reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); @@ -79,49 +60,6 @@ public class MetricsReporter extends Maintainer { } } - private void reportChefMetrics() { - String query = "chef_environment:hosted*"; - if (system == SystemName.cd) { - query += " AND hosted_system:" + system; - } - PartialNodeResult nodeResult = chefClient.partialSearchNodes(query, - List.of( - AttributeMapping.simpleMapping("fqdn"), - AttributeMapping.simpleMapping("ohai_time"), - AttributeMapping.deepMapping("tenant", List.of("hosted", "owner", "tenant")), - AttributeMapping.deepMapping("application", List.of("hosted", "owner", "application")), - AttributeMapping.deepMapping("instance", List.of("hosted", "owner", "instance")), - AttributeMapping.deepMapping("environment", List.of("hosted", "environment")), - AttributeMapping.deepMapping("region", List.of("hosted", "region")), - AttributeMapping.deepMapping("system", List.of("hosted", "system")) - )); - - // The above search will return a correct list if the system is CD. However for main, it will - // return all nodes, since system==nil for main - keepNodesWithSystem(nodeResult, system); - - Instant instant = clock.instant(); - for (PartialNode node : nodeResult.rows) { - String hostname = node.getFqdn(); - long secondsSinceConverge = Duration.between(Instant.ofEpochSecond(node.getOhaiTime().longValue()), instant).getSeconds(); - Map<String, String> dimensions = new HashMap<>(); - dimensions.put("host", hostname); - dimensions.put("system", node.getValue("system").orElse("main")); - Optional<String> environment = node.getValue("environment"); - Optional<String> region = node.getValue("region"); - - if (environment.isPresent() && region.isPresent()) { - dimensions.put("zone", String.format("%s.%s", environment.get(), region.get())); - } - - node.getValue("tenant").ifPresent(tenant -> dimensions.put("tenantName", tenant)); - Optional<String> application = node.getValue("application"); - application.ifPresent(app -> dimensions.put("app", String.format("%s.%s", app, node.getValue("instance").orElse("default")))); - Metric.Context context = metric.createContext(dimensions); - metric.set(CONVERGENCE_METRIC, secondsSinceConverge, context); - } - } - private void reportDeploymentMetrics() { ApplicationList applications = ApplicationList.from(controller().applications().asList()) .hasProductionDeployment(); @@ -210,10 +148,6 @@ public class MetricsReporter extends Maintainer { .max(Integer::compareTo) .orElse(0); } - - private static void keepNodesWithSystem(PartialNodeResult nodeResult, SystemName system) { - nodeResult.rows.removeIf(node -> !system.value().equals(node.getValue("system").orElse("main"))); - } private static Map<String, String> dimensions(ApplicationId application) { return ImmutableMap.of( diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 6ecf60e7404..0c045eb7253 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -49,6 +50,7 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Serializes {@link Application} to/from slime. @@ -551,23 +553,25 @@ public class ApplicationSerializer { } private List<AssignedRotation> assignedRotationsFromSlime(DeploymentSpec deploymentSpec, Inspector root) { - final var assignedRotations = new LinkedHashSet<AssignedRotation>(); + final var assignedRotations = new LinkedHashMap<EndpointId, AssignedRotation>(); // Add the legacy rotation field to the set - this needs to be first // TODO: Remove when we retire the rotations field final var legacyRotation = legacyRotationFromSlime(root.field(deprecatedRotationField)); if (legacyRotation.isPresent() && deploymentSpec.globalServiceId().isPresent()) { final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); - assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get())); + final var regions = deploymentSpec.zones().stream().flatMap(zone -> zone.region().stream()).collect(Collectors.toSet()); + assignedRotations.putIfAbsent(EndpointId.default_(), new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get(), regions)); } // Now add the same entries from "stupid" list of rotations // TODO: Remove when we retire the rotations field final var rotations = rotationListFromSlime(root.field(rotationsField)); for (var rotation : rotations) { + final var regions = deploymentSpec.zones().stream().flatMap(zone -> zone.region().stream()).collect(Collectors.toSet()); if (deploymentSpec.globalServiceId().isPresent()) { final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); - assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), rotation)); + assignedRotations.putIfAbsent(EndpointId.default_(), new AssignedRotation(clusterId, EndpointId.default_(), rotation, regions)); } } @@ -576,10 +580,14 @@ public class ApplicationSerializer { final var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString()); final var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString()); final var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString()); - assignedRotations.add(new AssignedRotation(clusterId, endpointId, rotationId)); + final var regions = deploymentSpec.endpoints().stream() + .filter(endpoint -> endpoint.endpointId().equals(endpointId.id())) + .flatMap(endpoint -> endpoint.regions().stream()) + .collect(Collectors.toSet()); + assignedRotations.putIfAbsent(endpointId, new AssignedRotation(clusterId, endpointId, rotationId, regions)); }); - return List.copyOf(assignedRotations); + return List.copyOf(assignedRotations.values()); } private List<RotationId> rotationListFromSlime(Inspector field) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 9f091061596..dce7d2e68ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -7,15 +7,13 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -50,7 +48,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; @@ -220,7 +217,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); @@ -238,11 +234,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -1120,38 +1114,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { "region", region)); } - /** - * Promote application Chef environments. To be used by component jobs only - */ - private HttpResponse promoteApplication(String tenantName, String applicationName, HttpRequest request) { - try{ - ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); - String sourceEnvironment = chefEnvironment.systemChefEnvironment(); - String targetEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); - controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); - return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); - } catch (Exception e) { - log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s)", tenantName, applicationName), e); - return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); - } - } - - /** - * Promote application Chef environments for jobs that deploy applications - */ - private HttpResponse promoteApplicationDeployment(String tenantName, String applicationName, String environmentName, String regionName, String instanceName, HttpRequest request) { - try { - ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); - String sourceEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); - String targetEnvironment = chefEnvironment.applicationTargetEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName), Environment.from(environmentName), RegionName.from(regionName)); - controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); - return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); - } catch (Exception e) { - log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s %s.%s)", tenantName, applicationName, environmentName, regionName), e); - return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); - } - } - private HttpResponse notifyJobCompletion(String tenant, String application, HttpRequest request) { try { DeploymentJobs.JobReport report = toJobReport(tenant, application, toSlime(request.getData()).get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java deleted file mode 100644 index 7c32e48e218..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; - -/** - * Represents Chef environments for applications/deployments. Used for promotion of Chef environments - * - * @author mortent - */ -public class ApplicationChefEnvironment { - - private final String systemChefEnvironment; - private final String systemSuffix; - - public ApplicationChefEnvironment(SystemName system) { - if (system == SystemName.main) { - systemChefEnvironment = "hosted-verified-prod"; - systemSuffix = ""; - } else { - systemChefEnvironment = "hosted-infra-cd"; - systemSuffix = "-cd"; - } - } - - public String systemChefEnvironment() { - return systemChefEnvironment; - } - - public String applicationSourceEnvironment(TenantName tenantName, ApplicationName applicationName) { - // placeholder and component already used in legacy chef promotion - return String.format("hosted-instance%s_%s_%s_placeholder_component_default", systemSuffix, tenantName, applicationName); - } - - public String applicationTargetEnvironment(TenantName tenantName, ApplicationName applicationName, Environment environment, RegionName regionName) { - return String.format("hosted-instance%s_%s_%s_%s_%s_default", systemSuffix, tenantName, applicationName, regionName, environment); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index d2b16721503..f2bc50ec445 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -1,18 +1,30 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.Endpoint; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -50,6 +62,11 @@ public class RotationRepository { return application.rotations().stream().map(allRotations::get).findFirst(); } + /** Get rotation for the given rotationId */ + public Optional<Rotation> getRotation(RotationId rotationId) { + return Optional.of(allRotations.get(rotationId)); + } + /** * Returns a rotation for the given application * @@ -76,6 +93,116 @@ public class RotationRepository { } /** + * Returns rotation assignments for all endpoints in application. + * + * If rotations are already assigned, these will be returned. + * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. + * This method supports both global-service-id as well as the new endpoints tag. + * + * @param application The application requesting rotations. + * @param lock Lock which by acquired by the caller + * @return List of rotation assignments - either new or existing. + */ + public List<AssignedRotation> getOrAssignRotations(Application application, RotationLock lock) { + if (application.deploymentSpec().globalServiceId().isPresent() && ! application.deploymentSpec().endpoints().isEmpty()) { + throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); + } + + // Support the older case of setting global-service-id + if (application.deploymentSpec().globalServiceId().isPresent()) { + final var regions = application.deploymentSpec().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + + final var rotation = getOrAssignRotation(application, lock); + + return List.of( + new AssignedRotation( + new ClusterSpec.Id(application.deploymentSpec().globalServiceId().get()), + EndpointId.default_(), + rotation.id(), + regions + ) + ); + } + + final Map<EndpointId, AssignedRotation> existingAssignments = existingEndpointAssignments(application); + final Map<EndpointId, AssignedRotation> updatedAssignments = assignRotationsToEndpoints(application, existingAssignments, lock); + + existingAssignments.putAll(updatedAssignments); + + return List.copyOf(existingAssignments.values()); + } + + private Map<EndpointId, AssignedRotation> assignRotationsToEndpoints(Application application, Map<EndpointId, AssignedRotation> existingAssignments, RotationLock lock) { + final var availableRotations = new ArrayList<>(availableRotations(lock).values()); + + final var neededRotations = application.deploymentSpec().endpoints().stream() + .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) + .collect(Collectors.toSet()); + + if (neededRotations.size() > availableRotations.size()) { + throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation: need " + neededRotations.size() + ", have " + availableRotations.size()); + } + + return neededRotations.stream() + .map(endpoint -> { + return new AssignedRotation( + new ClusterSpec.Id(endpoint.containerId()), + EndpointId.of(endpoint.endpointId()), + availableRotations.remove(0).id(), + endpoint.regions() + ); + }) + .collect( + Collectors.toMap( + AssignedRotation::endpointId, + Function.identity(), + (a, b) -> { throw new IllegalStateException("Duplicate entries:" + a + ", " + b); }, + LinkedHashMap::new + ) + ); + } + + private Map<EndpointId, AssignedRotation> existingEndpointAssignments(Application application) { + // + // Get the regions that has been configured for an endpoint. Empty set if the endpoint + // is no longer mentioned in the configuration file. + // + final Function<EndpointId, Set<RegionName>> configuredRegionsForEndpoint = endpointId -> { + return application.deploymentSpec().endpoints().stream() + .filter(endpoint -> endpointId.id().equals(endpoint.endpointId())) + .map(Endpoint::regions) + .findFirst() + .orElse(Set.of()); + }; + + // + // Build a new AssignedRotation instance where we update set of regions from the configuration instead + // of using the one already mentioned in the assignment. This allows us to overwrite the set of regions + // when + final Function<AssignedRotation, AssignedRotation> assignedRotationWithConfiguredRegions = assignedRotation -> { + return new AssignedRotation( + assignedRotation.clusterId(), + assignedRotation.endpointId(), + assignedRotation.rotationId(), + configuredRegionsForEndpoint.apply(assignedRotation.endpointId()) + ); + }; + + return application.assignedRotations().stream() + .collect( + Collectors.toMap( + AssignedRotation::endpointId, + assignedRotationWithConfiguredRegions, + (a, b) -> { throw new IllegalStateException("Duplicate entries: " + a + ", " + b); }, + LinkedHashMap::new + ) + ); + } + + /** * Returns all unassigned rotations * @param lock Lock which must be acquired by the caller */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index de31f1f67f9..c26f1879f6a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -23,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevisi import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -275,6 +277,8 @@ public class ControllerTest { @Test public void testDnsAliasRegistration() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -290,12 +294,42 @@ public class ControllerTest { for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), Set.of("rotation-id-01", - "app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.yahooapis.com", - "app1--tenant1.global.vespa.yahooapis.com"), + "app1--tenant1.global.vespa.oath.cloud"), tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); } tester.flushDnsRequests(); + + assertEquals(1, tester.controllerTester().nameService().records().size()); + + var record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record.isPresent()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); + assertEquals("rotation-fqdn-01.", record.get().data().asString()); + } + + @Test + public void testDnsAliasRegistrationLegacy() { + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .globalServiceId("foo") + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage); + Collection<Deployment> deployments = tester.application(application.id()).deployments().values(); + assertFalse(deployments.isEmpty()); + for (Deployment deployment : deployments) { + assertEquals("Rotation names are passed to config server in " + deployment.zone(), + Set.of("rotation-id-01", + "app1--tenant1.global.vespa.oath.cloud", + "app1.tenant1.global.vespa.yahooapis.com", + "app1--tenant1.global.vespa.yahooapis.com"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); + } + tester.flushDnsRequests(); assertEquals(3, tester.controllerTester().nameService().records().size()); Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); @@ -315,38 +349,134 @@ public class ControllerTest { } @Test - public void testRedirectLegacyDnsNames() { // TODO: Remove together with Flags.REDIRECT_LEGACY_DNS_NAMES + public void testDnsAliasRegistrationWithEndpoints() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) - .globalServiceId("foo") + .endpoint("foobar", "qrs", "us-west-1", "us-central-1") + .endpoint("default", "qrs", "us-west-1", "us-central-1") .region("us-west-1") .region("us-central-1") .build(); - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.REDIRECT_LEGACY_DNS_NAMES.id(), true); + tester.deployCompletely(application, applicationPackage); + Collection<Deployment> deployments = tester.application(application.id()).deployments().values(); + assertFalse(deployments.isEmpty()); + for (Deployment deployment : deployments) { + assertEquals("Rotation names are passed to config server in " + deployment.zone(), + Set.of( + "rotation-id-01", + "rotation-id-02", + "app1--tenant1.global.vespa.oath.cloud", + "foobar--app1--tenant1.global.vespa.oath.cloud" + ), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); + } + tester.flushDnsRequests(); + + assertEquals(2, tester.controllerTester().nameService().records().size()); + + var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record1.isPresent()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record1.get().name().asString()); + assertEquals("rotation-fqdn-02.", record1.get().data().asString()); + + var record2 = tester.controllerTester().findCname("foobar--app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record2.isPresent()); + assertEquals("foobar--app1--tenant1.global.vespa.oath.cloud", record2.get().name().asString()); + assertEquals("rotation-fqdn-01.", record2.get().data().asString()); + } + + @Test + public void testDnsAliasRegistrationWithChangingZones() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1", "us-central-1") + .region("us-west-1") + .region("us-central-1") + .build(); tester.deployCompletely(application, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); - Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("app1--tenant1.global.vespa.oath.cloud.", record.get().data().asString()); + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); - record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-central-1"))) + ); - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("app1--tenant1.global.vespa.oath.cloud.", record.get().data().asString()); + + ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1") + .region("us-west-1") + .region("us-central-1") + .build(); + + tester.deployCompletely(application, applicationPackage2, BuildJob.defaultBuildNumber + 1); + + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); + + assertEquals( + Set.of(), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-central-1"))) + ); + + assertEquals(Set.of(RegionName.from("us-west-1")), tester.application(application.id()).assignedRotations().get(0).regions()); } @Test + public void testUnassignRotations() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1", "us-central-1") + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage); + + ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage2, BuildJob.defaultBuildNumber + 1); + + + assertEquals( + List.of(AssignedRotation.fromStrings("qrs", "default", "rotation-id-01", Set.of())), + tester.application(application.id()).assignedRotations() + ); + + assertEquals( + Set.of(), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); + } + + @Test public void testUpdatesExistingDnsAlias() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + // Application 1 is deployed and deleted { Application app1 = tester.createApplication("app1", "tenant1", 1, 1L); @@ -358,16 +488,11 @@ public class ControllerTest { .build(); tester.deployCompletely(app1, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); - - Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals(1, tester.controllerTester().nameService().records().size()); - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); + Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); // Application is deleted and rotation is unassigned @@ -408,22 +533,12 @@ public class ControllerTest { .region("us-central-1") .build(); tester.deployCompletely(app2, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); - - Optional<Record> record = tester.controllerTester().findCname("app2--tenant2.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app2--tenant2.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals(1, tester.controllerTester().nameService().records().size()); - record = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); + var record = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); assertTrue(record.isPresent()); assertEquals("app2--tenant2.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app2.tenant2.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app2.tenant2.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); } // Application 1 is recreated, deployed and assigned a new rotation @@ -441,19 +556,15 @@ public class ControllerTest { assertEquals("rotation-id-02", app1.rotations().get(0).asString()); // DNS records are created for the newly assigned rotation - assertEquals(6, tester.controllerTester().nameService().records().size()); + assertEquals(2, tester.controllerTester().nameService().records().size()); - Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); + var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record1.isPresent()); + assertEquals("rotation-fqdn-02.", record1.get().data().asString()); - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); + var record2 = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); + assertTrue(record2.isPresent()); + assertEquals("rotation-fqdn-01.", record2.get().data().asString()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index dbf983a5bab..d5935c752d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; -import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -29,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; -import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; @@ -100,7 +98,7 @@ public final class ControllerTester { this(new AthenzDbMock(), clock, new ConfigServerMock(new ZoneRegistryMock()), new ZoneRegistryMock(), new GitHubMock(), curatorDb, rotationsConfig, new MemoryNameService(), new ArtifactRepositoryMock(), new ApplicationStoreMock(), new MockBuildService(), - metricsService, new RoutingGeneratorMock(), new MockContactRetriever(), new MockIssueHandler(clock)); + metricsService, new RoutingGeneratorMock(), new MockContactRetriever()); } public ControllerTester(ManualClock clock) { @@ -125,7 +123,7 @@ public final class ControllerTester { MemoryNameService nameService, ArtifactRepositoryMock artifactRepository, ApplicationStoreMock appStoreMock, MockBuildService buildService, MetricsServiceMock metricsService, RoutingGeneratorMock routingGenerator, - MockContactRetriever contactRetriever, MockIssueHandler issueHandler) { + MockContactRetriever contactRetriever) { this.athenzDb = athenzDb; this.clock = clock; this.configServer = configServer; @@ -141,7 +139,7 @@ public final class ControllerTester { this.routingGenerator = routingGenerator; this.contactRetriever = contactRetriever; this.controller = createController(curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, - athenzDb, nameService, artifactRepository, appStoreMock, buildService, + athenzDb, artifactRepository, appStoreMock, buildService, metricsService, routingGenerator); // Make root logger use time from manual clock @@ -199,7 +197,7 @@ public final class ControllerTester { /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ public final void createNewController() { controller = createController(curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, athenzDb, - nameService, artifactRepository, applicationStore, buildService, metricsService, + artifactRepository, applicationStore, buildService, metricsService, routingGenerator); } @@ -332,7 +330,7 @@ public final class ControllerTester { private static Controller createController(CuratorDb curator, RotationsConfig rotationsConfig, ConfigServerMock configServer, ManualClock clock, GitHubMock gitHub, ZoneRegistryMock zoneRegistryMock, - AthenzDbMock athensDb, MemoryNameService nameService, + AthenzDbMock athensDb, ArtifactRepository artifactRepository, ApplicationStore applicationStore, BuildService buildService, MetricsServiceMock metricsService, RoutingGenerator routingGenerator) { @@ -343,7 +341,6 @@ public final class ControllerTester { configServer, metricsService, routingGenerator, - new ChefMock(), clock, new AthenzFacade(new AthenzClientFactoryMock(athensDb)), artifactRepository, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index 16b875c1892..f5047a82e2f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -66,6 +66,50 @@ public class EndpointTest { } @Test + public void test_global_endpoints_with_endpoint_id() { + final var endpointId = EndpointId.default_(); + + Map<String, Endpoint> tests = Map.of( + // Legacy endpoint + "http://a1.t1.global.vespa.yahooapis.com:4080/", + Endpoint.of(app1).named(endpointId).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Legacy endpoint with TLS + "https://a1--t1.global.vespa.yahooapis.com:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Main endpoint + "https://a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.main), + + // Main endpoint in CD + "https://cd--a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.cd), + + // Main endpoint with direct routing and default TLS port + "https://a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint with custom rotation name + "https://r1.a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance in default rotation + "https://a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance with custom rotation name + "https://r2.a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint in public system + "https://a1.t1.global.public.vespa.oath.cloud/", + Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + + @Test public void test_zone_endpoints() { ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing ZoneId prodZone = ZoneId.from("prod", "us-north-1"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 83b95ccc8b0..6635547e9be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -17,6 +17,9 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.OptionalInt; import java.util.StringJoiner; import java.util.zip.ZipEntry; @@ -35,6 +38,7 @@ public class ApplicationPackageBuilder { private final StringJoiner notifications = new StringJoiner("/>\n <email ", "<notifications>\n <email ", "/>\n</notifications>\n").setEmptyValue(""); + private final StringBuilder endpointsBody = new StringBuilder(); private OptionalInt majorVersion = OptionalInt.empty(); private String upgradePolicy = null; @@ -63,6 +67,18 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder endpoint(String endpointId, String containerId, String... regions) { + endpointsBody.append(" <endpoint"); + endpointsBody.append(" id='").append(endpointId).append("'"); + endpointsBody.append(" container-id='").append(containerId).append("'"); + endpointsBody.append(">\n"); + for (var region : regions) { + endpointsBody.append(" <region>").append(region).append("</region>\n"); + } + endpointsBody.append(" </endpoint>\n"); + return this; + } + public ApplicationPackageBuilder region(RegionName regionName) { return region(regionName.value()); } @@ -157,7 +173,11 @@ public class ApplicationPackageBuilder { xml.append(environmentBody); xml.append(" </"); xml.append(environment.value()); - xml.append(">\n</deployment>"); + xml.append(">\n"); + xml.append(" <endpoints>\n"); + xml.append(endpointsBody); + xml.append(" </endpoints>\n"); + xml.append("</deployment>"); return xml.toString().getBytes(StandardCharsets.UTF_8); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index cdbc45c4d8f..a89c5988396 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -45,6 +45,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; /** * @author mortent @@ -226,7 +227,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationNames, - List<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content) { + Set<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); if (prepareException != null) { RuntimeException prepareException = this.prepareException; @@ -238,7 +239,13 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty()) provision(deployment.zoneId(), deployment.applicationId()); - this.rotationNames.put(deployment, Set.copyOf(rotationNames)); + this.rotationNames.put( + deployment, + Stream.concat( + containerEndpoints.stream().flatMap(e -> e.names().stream()), + rotationNames.stream() + ).collect(Collectors.toSet()) + ); return () -> { Application application = applications.get(deployment.applicationId()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 58f35c0ac05..148be3f258e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -1,38 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; -import com.yahoo.test.ManualClock; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; -import com.yahoo.vespa.hosted.controller.integration.MetricsMock.MapContext; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Clock; import java.time.Duration; -import java.time.Instant; -import java.util.Map; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; @@ -40,39 +25,13 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; /** * @author mortent */ public class MetricsReporterTest { - private static final Path testData = Paths.get("src/test/resources/"); - - private MetricsMock metrics; - - @Before - public void before() { - metrics = new MetricsMock(); - } - - @Test - public void test_chef_metrics() { - Clock clock = new ManualClock(Instant.ofEpochSecond(1475497913)); - ControllerTester tester = new ControllerTester(); - MetricsReporter metricsReporter = createReporter(clock, tester.controller(), metrics, SystemName.cd); - metricsReporter.maintain(); - assertEquals(2, metrics.getMetrics().size()); - - Map<MapContext, Map<String, Number>> hostMetrics = getMetricsByHost("fake-node.test"); - assertEquals(1, hostMetrics.size()); - Map.Entry<MapContext, Map<String, Number>> metricEntry = hostMetrics.entrySet().iterator().next(); - MapContext metricContext = metricEntry.getKey(); - assertDimension(metricContext, "tenantName", "ciintegrationtests"); - assertDimension(metricContext, "app", "restart.default"); - assertDimension(metricContext, "zone", "prod.cd-us-east-1"); - assertEquals(727, metricEntry.getValue().get(MetricsReporter.CONVERGENCE_METRIC).longValue()); - } + private final MetricsMock metrics = new MetricsMock(); @Test public void test_deployment_fail_ratio() { @@ -81,7 +40,7 @@ public class MetricsReporterTest { .environment(Environment.prod) .region("us-west-1") .build(); - MetricsReporter metricsReporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter metricsReporter = createReporter(tester.controller()); metricsReporter.maintain(); assertEquals(0.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC)); @@ -108,14 +67,6 @@ public class MetricsReporterTest { } @Test - public void test_chef_metrics_omit_zone_when_unknown() { - ControllerTester tester = new ControllerTester(); - String hostname = "fake-node2.test"; - MapContext metricContext = getMetricContextByHost(tester.controller(), hostname); - assertNull(metricContext.getDimensions().get("zone")); - } - - @Test public void test_deployment_average_duration() { DeploymentTester tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -123,7 +74,7 @@ public class MetricsReporterTest { .region("us-west-1") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.deployCompletely(app, applicationPackage); @@ -165,7 +116,7 @@ public class MetricsReporterTest { .region("us-west-1") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application app = tester.createApplication("app1", "tenant1", 1, 11L); // Initial deployment without failures @@ -216,7 +167,7 @@ public class MetricsReporterTest { .region("us-west-1") .region("us-east-3") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application application = tester.createApplication("app1", "tenant1", 1, 11L); tester.configServer().generateWarnings(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1")), 3); tester.configServer().generateWarnings(new DeploymentId(application.id(), ZoneId.from("prod", "us-east-3")), 4); @@ -231,7 +182,7 @@ public class MetricsReporterTest { ApplicationVersion version = tester.deployNewSubmission(); assertEquals(1000, version.buildTime().get().toEpochMilli()); - MetricsReporter reporter = createReporter(tester.tester().controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.tester().controller()); reporter.maintain(); assertEquals(tester.clock().instant().getEpochSecond() - 1, getMetric(MetricsReporter.DEPLOYMENT_BUILD_AGE_SECONDS, tester.app())); @@ -246,7 +197,7 @@ public class MetricsReporterTest { .region("us-west-1") .region("us-east-3") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application application = tester.createApplication("app1", "tenant1", 1, 11L); reporter.maintain(); assertEquals("Queue is empty initially", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); @@ -279,43 +230,8 @@ public class MetricsReporterTest { .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + application.id())); } - private MetricsReporter createReporter(Controller controller, MetricsMock metricsMock, SystemName system) { - return createReporter(controller.clock(), controller, metricsMock, system); - } - - private MetricsReporter createReporter(Clock clock, Controller controller, MetricsMock metricsMock, - SystemName system) { - ChefMock chef = new ChefMock(); - PartialNodeResult result; - try { - result = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .readValue(testData.resolve("chef_output.json").toFile(), PartialNodeResult.class); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - chef.addPartialResult(result.rows); - return new MetricsReporter(controller, metricsMock, chef, clock, new JobControl(new MockCuratorDb()), system); - } - - private Map<MapContext, Map<String, Number>> getMetricsByHost(String hostname) { - return metrics.getMetrics((dimensions) -> hostname.equals(dimensions.get("host"))); - } - - private MapContext getMetricContextByHost(Controller controller, String hostname) { - MetricsReporter metricsReporter = createReporter(controller, metrics, SystemName.main); - metricsReporter.maintain(); - - assertFalse(metrics.getMetrics().isEmpty()); - - Map<MapContext, Map<String, Number>> metrics = getMetricsByHost(hostname); - assertEquals(1, metrics.size()); - Map.Entry<MapContext, Map<String, Number>> metricEntry = metrics.entrySet().iterator().next(); - return metricEntry.getKey(); - } - - private static void assertDimension(MapContext metricContext, String dimensionName, String expectedValue) { - assertEquals(expectedValue, metricContext.getDimensions().get(dimensionName)); + private MetricsReporter createReporter(Controller controller) { + return new MetricsReporter(controller, metrics, new JobControl(new MockCuratorDb())); } private static String appDimension(Application application) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 347fe6064df..7b39b0d53a4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; @@ -46,6 +47,7 @@ import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; +import java.util.Set; import java.util.TreeMap; import static com.yahoo.config.provision.SystemName.main; @@ -119,7 +121,7 @@ public class ApplicationSerializerTest { OptionalInt.of(7), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), - List.of(new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.default_(), new RotationId("my-rotation"))), + List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of())), rotationStatus, Optional.of(new ApplicationCertificate("vespa.certificate"))); @@ -258,6 +260,11 @@ public class ApplicationSerializerTest { final var applicationJson = Files.readAllBytes(testData.resolve("complete-application.json")); final var slime = SlimeUtils.jsonToSlime(applicationJson); + final var regions = Set.of( + RegionName.from("us-east-3"), + RegionName.from("us-west-1") + ); + // Add the necessary fields to the Slime representation of the application final var cursor = slime.get(); cursor.setString("rotation", "single-rotation"); @@ -275,11 +282,11 @@ public class ApplicationSerializerTest { // Parse and test the output from parsing contains both legacy rotation and multiple rotations final var application = applicationSerializer.fromSlime(slime); + // Since only one AssignedEndpoint can be "default", we make sure that we are ignoring the + // multiple-rotation entries as the globalServiceId will override them assertEquals( List.of( new RotationId("single-rotation"), - new RotationId("multiple-rotation-1"), - new RotationId("multiple-rotation-2"), new RotationId("assigned-rotation") ), application.rotations() @@ -289,12 +296,13 @@ public class ApplicationSerializerTest { Optional.of(new RotationId("single-rotation")), application.legacyRotation() ); + // The same goes here for AssignedRotations with "default" EndpointId as in the .rotations() test above. + // Note that we are only using Set.of() on "assigned-rotation" because in this test we do not have access + // to a deployment.xml that describes the zones a rotation should map to. assertEquals( List.of( - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("single-rotation")), - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("multiple-rotation-1")), - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("multiple-rotation-2")), - new AssignedRotation(new ClusterSpec.Id("foobar"), EndpointId.of("nice-endpoint"), new RotationId("assigned-rotation")) + new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("single-rotation"), regions), + new AssignedRotation(new ClusterSpec.Id("foobar"), EndpointId.of("nice-endpoint"), new RotationId("assigned-rotation"), Set.of()) ), application.assignedRotations() ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 53476a2e42f..797d2b9aa0e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -62,7 +62,6 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock'/>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 16fd10277d2..a0acbc625f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -16,7 +16,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -61,7 +60,6 @@ import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import org.junit.Before; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json deleted file mode 100644 index fef3cf6a372..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "application": "tenant1:application1:default", - "zone": "dev.us-east-1", - "system": "main", - "endpoints": { - "dev.us-east-1": [ - "http://old-endpoint.vespa.yahooapis.com:4080" - ], - "prod.us-central-1": [ - "http://old-endpoint.vespa.yahooapis.com:4080" - ] - }, - "zoneEndpoints": { - "dev.us-east-1": { - "default": "http://old-endpoint.vespa.yahooapis.com:4080" - }, - "prod.us-central-1": { - "default": "http://old-endpoint.vespa.yahooapis.com:4080" - } - } -} diff --git a/controller-server/src/test/resources/chef_output.json b/controller-server/src/test/resources/chef_output.json deleted file mode 100644 index 257065f7b5b..00000000000 --- a/controller-server/src/test/resources/chef_output.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "total": 1, - "start": 0, - "rows": [ - { - "url": "https://chef-server.test/organizations/vespa/nodes/fake-node.test", - "data": { - "fqdn": "fake-node.test", - "ohai_time": 1475497186.68962, - "tenant": "ciintegrationtests", - "application": "restart", - "instance": "default", - "zone": "cd_cd-us-east-1_prod", - "system": "cd", - "environment": "prod", - "region": "cd-us-east-1" - } - }, - { - "url": "https://chef-server.test/organizations/vespa/nodes/fake-node2.test", - "data": { - "fqdn": "fake-node2.test", - "ohai_time": 1475497186.68962, - "tenant": null, - "application": null, - "instance": null, - "zone": null, - "system": null, - "environment": null, - "region": null - } - } - ] -} diff --git a/controller-server/src/test/resources/job-grandparent.json b/controller-server/src/test/resources/job-grandparent.json deleted file mode 100644 index 63602bed146..00000000000 --- a/controller-server/src/test/resources/job-grandparent.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "duration": 720000, - "causes": [] -} diff --git a/controller-server/src/test/resources/job-parent.json b/controller-server/src/test/resources/job-parent.json deleted file mode 100644 index 88d50de394f..00000000000 --- a/controller-server/src/test/resources/job-parent.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "duration": 1200000, - "causes": [ - { - "upstreamBuild": 231, - "upstreamProject": "3-v3-job-grandparent" - } - ] -} diff --git a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp index d970b9dad30..356625417d8 100644 --- a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp @@ -102,7 +102,7 @@ EvalFixture::ParamRepo make_params() { .add("v04_y3", spec({y(3)}, MyVecSeq(10))) .add("v05_x5", spec({x(5)}, MyVecSeq(6.0))) .add("v06_x5", spec({x(5)}, MyVecSeq(7.0))) - .add("v07_x5f", spec({x(5)}, MyVecSeq(7.0)), "tensor<float>(x[5])") + .add("v07_x5f", spec(float_cells({x(5)}), MyVecSeq(7.0))) .add("m01_x3y3", spec({x(3),y(3)}, MyVecSeq(1.0))) .add("m02_x3y3", spec({x(3),y(3)}, MyVecSeq(2.0))); } diff --git a/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp b/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp index 773381b4c77..4995ea89735 100644 --- a/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp +++ b/eval/src/tests/tensor/dense_fast_rename_optimizer/dense_fast_rename_optimizer_test.cpp @@ -25,7 +25,7 @@ const TensorEngine &prod_engine = DefaultTensorEngine::ref(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("x5", spec({x(5)}, N())) - .add("x5f", spec({x(5)}, N()), "tensor<float>(x[5])") + .add("x5f", spec(float_cells({x(5)}), N())) .add("x_m", spec({x({"a", "b", "c"})}, N())) .add("x5y3", spec({x(5),y(3)}, N())); } diff --git a/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp b/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp index c9e581e6b21..083ed1c7071 100644 --- a/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp +++ b/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp @@ -45,8 +45,8 @@ EvalFixture::ParamRepo make_params() { .add_mutable("mut_x5_A", spec({x(5)}, seq)) .add_mutable("mut_x5_B", spec({x(5)}, seq)) .add_mutable("mut_x5_C", spec({x(5)}, seq)) - .add_mutable("mut_x5f_D", spec({x(5)}, seq), "tensor<float>(x[5])") - .add_mutable("mut_x5f_E", spec({x(5)}, seq), "tensor<float>(x[5])") + .add_mutable("mut_x5f_D", spec(float_cells({x(5)}), seq)) + .add_mutable("mut_x5f_E", spec(float_cells({x(5)}), seq)) .add_mutable("mut_x5y3_A", spec({x(5),y(3)}, seq)) .add_mutable("mut_x5y3_B", spec({x(5),y(3)}, seq)) .add_mutable("mut_x_sparse", spec({x({"a", "b", "c"})}, seq)); diff --git a/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp b/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp index 36ebdec028b..314d3a6186c 100644 --- a/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp +++ b/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp @@ -26,7 +26,7 @@ EvalFixture::ParamRepo make_params() { .add("x5", spec({x(5)}, N())) .add_mutable("_d", spec(5.0)) .add_mutable("_x5", spec({x(5)}, N())) - .add_mutable("_x5f", spec({x(5)}, N()), "tensor<float>(x[5])") + .add_mutable("_x5f", spec(float_cells({x(5)}), N())) .add_mutable("_x5y3", spec({x(5),y(3)}, N())) .add_mutable("_x_m", spec({x({"a", "b", "c"})}, N())); } diff --git a/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp b/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp index 65208aedb4b..7856775ae30 100644 --- a/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp +++ b/eval/src/tests/tensor/dense_remove_dimension_optimizer/dense_remove_dimension_optimizer_test.cpp @@ -25,7 +25,7 @@ const TensorEngine &prod_engine = DefaultTensorEngine::ref(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("x1y5z1", spec({x(1),y(5),z(1)}, N())) - .add("x1y5z1f", spec({x(1),y(5),z(1)}, N()), "tensor<float>(x[1],y[5],z[1])") + .add("x1y5z1f", spec(float_cells({x(1),y(5),z(1)}), N())) .add("x1y1z1", spec({x(1),y(1),z(1)}, N())) .add("x1y5z_m", spec({x(1),y(5),z({"a"})}, N())); } diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp index 8045958d9ba..335aa4791a4 100644 --- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp @@ -38,13 +38,13 @@ EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("y1", spec({y(1)}, MyVecSeq())) .add("y3", spec({y(3)}, MyVecSeq())) - .add("y3f", spec({y(3)}, MyVecSeq()), "tensor<float>(y[3])") + .add("y3f", spec(float_cells({y(3)}), MyVecSeq())) .add("y5", spec({y(5)}, MyVecSeq())) .add("y16", spec({y(16)}, MyVecSeq())) .add("x1y1", spec({x(1),y(1)}, MyMatSeq())) .add("y1z1", spec({y(1),z(1)}, MyMatSeq())) .add("x2y3", spec({x(2),y(3)}, MyMatSeq())) - .add("x2y3f", spec({x(2),y(3)}, MyMatSeq()), "tensor<float>(x[2],y[3])") + .add("x2y3f", spec(float_cells({x(2),y(3)}), MyMatSeq())) .add("x2z3", spec({x(2),z(3)}, MyMatSeq())) .add("y3z2", spec({y(3),z(2)}, MyMatSeq())) .add("x8y5", spec({x(8),y(5)}, MyMatSeq())) diff --git a/eval/src/vespa/eval/eval/tensor_spec.h b/eval/src/vespa/eval/eval/tensor_spec.h index 32dc1c82fcb..25af4c7a93c 100644 --- a/eval/src/vespa/eval/eval/tensor_spec.h +++ b/eval/src/vespa/eval/eval/tensor_spec.h @@ -73,7 +73,6 @@ public: } return *this; } - void override_type(const vespalib::string &new_type) { _type = new_type; } const vespalib::string &type() const { return _type; } const Cells &cells() const { return _cells; } vespalib::string to_string() const; diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index 321b472a3fa..3f5fa4d72bb 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -14,7 +14,7 @@ NodeTypes get_types(const Function &function, const ParamRepo ¶m_repo) { for (size_t i = 0; i < function.num_params(); ++i) { auto pos = param_repo.map.find(function.param_name(i)); ASSERT_TRUE(pos != param_repo.map.end()); - param_types.push_back(ValueType::from_spec(pos->second.type)); + param_types.push_back(ValueType::from_spec(pos->second.value.type())); ASSERT_TRUE(!param_types.back().is_error()); } return NodeTypes(function, param_types); diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index 8c7d15e7416..9c793c01861 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -18,32 +18,24 @@ class EvalFixture public: struct Param { TensorSpec value; // actual parameter value - vespalib::string type; // pre-defined type (could be abstract) bool is_mutable; // input will be mutable (if allow_mutable is true) - Param(TensorSpec value_in, const vespalib::string &type_in, bool is_mutable_in) - : value(std::move(value_in)), type(type_in), is_mutable(is_mutable_in) {} + Param(TensorSpec value_in, bool is_mutable_in) + : value(std::move(value_in)), is_mutable(is_mutable_in) {} ~Param() {} }; struct ParamRepo { std::map<vespalib::string,Param> map; ParamRepo() : map() {} - ParamRepo &add(const vespalib::string &name, TensorSpec value_in, const vespalib::string &type_in, bool is_mutable_in) { - value_in.override_type(type_in); - map.insert_or_assign(name, Param(std::move(value_in), type_in, is_mutable_in)); + ParamRepo &add(const vespalib::string &name, TensorSpec value_in, bool is_mutable_in) { + map.insert_or_assign(name, Param(std::move(value_in), is_mutable_in)); return *this; } - ParamRepo &add(const vespalib::string &name, TensorSpec value, const vespalib::string &type) { - return add(name, value, type, false); - } - ParamRepo &add_mutable(const vespalib::string &name, TensorSpec value, const vespalib::string &type) { - return add(name, value, type, true); - } ParamRepo &add(const vespalib::string &name, const TensorSpec &value) { - return add(name, value, value.type(), false); + return add(name, value, false); } ParamRepo &add_mutable(const vespalib::string &name, const TensorSpec &value) { - return add(name, value, value.type(), true); + return add(name, value, true); } ~ParamRepo() {} }; diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index 1e1bd828d41..16005970817 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -301,10 +301,13 @@ struct TestContext { TEST_DO(verify_create_type("double")); TEST_DO(verify_create_type("tensor(x{})")); TEST_DO(verify_create_type("tensor(x{},y{})")); + TEST_DO(verify_create_type("tensor<float>(x{},y{})")); TEST_DO(verify_create_type("tensor(x[5])")); TEST_DO(verify_create_type("tensor(x[5],y[10])")); + TEST_DO(verify_create_type("tensor<float>(x[5],y[10])")); TEST_DO(verify_create_type("tensor(x{},y[10])")); - TEST_DO(verify_create_type("tensor(x[5],y{})")); + TEST_DO(verify_create_type("tensor(x[5],y{})")); + TEST_DO(verify_create_type("tensor<float>(x[5],y{})")); } //------------------------------------------------------------------------- @@ -318,11 +321,14 @@ struct TestContext { {x(3)}, {x(3),y(5)}, {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar"})}, {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})} + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; for (const Layout &layout: layouts) { TensorSpec input = spec(layout, seq); @@ -363,11 +369,14 @@ struct TestContext { {x(3)}, {x(3),y(5)}, {x(3),y(5),z(7)}, + float_cells({x(3),y(5),z(7)}), {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar"})}, {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})} + {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; for (const Layout &layout: layouts) { TEST_DO(verify_result(eval.eval(engine, spec(layout, seq)), spec(layout, OpSeq(seq, ref_op)))); @@ -612,20 +621,29 @@ struct TestContext { void test_apply_op(const Eval &eval, join_fun_t op, const Sequence &seq) { std::vector<Layout> layouts = { - {}, {}, - {x(5)}, {x(5)}, - {x(5)}, {y(5)}, - {x(5)}, {x(5),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(3),y(5)}, {y(5),z(7)}, - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {y({"foo","bar","baz"})}, - {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, - {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})} + {}, {}, + {x(5)}, {x(5)}, + {x(5)}, {y(5)}, + {x(5)}, {x(5),y(5)}, + {y(3)}, {x(2),z(3)}, + {x(3),y(5)}, {y(5),z(7)}, + float_cells({x(3),y(5)}), {y(5),z(7)}, + {x(3),y(5)}, float_cells({y(5),z(7)}), + float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), + {x({"a","b","c"})}, {x({"a","b","c"})}, + {x({"a","b","c"})}, {x({"a","b"})}, + {x({"a","b","c"})}, {y({"foo","bar","baz"})}, + {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, + {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), + {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, + {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, + float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, + {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), + float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) }; ASSERT_TRUE((layouts.size() % 2) == 0); for (size_t i = 0; i < layouts.size(); i += 2) { @@ -681,10 +699,20 @@ struct TestContext { TEST_DO(verify_result(safe(eval).eval(engine, lhs, rhs), spec(expect))); } + void test_dot_product(double expect, + const Layout &lhs, const Seq &lhs_seq, + const Layout &rhs, const Seq &rhs_seq) + { + TEST_DO(test_dot_product(expect, spec(lhs, lhs_seq), spec(rhs, rhs_seq))); + TEST_DO(test_dot_product(expect, spec(float_cells(lhs), lhs_seq), spec(rhs, rhs_seq))); + TEST_DO(test_dot_product(expect, spec(lhs, lhs_seq), spec(float_cells(rhs), rhs_seq))); + TEST_DO(test_dot_product(expect, spec(float_cells(lhs), lhs_seq), spec(float_cells(rhs), rhs_seq))); + } + void test_dot_product() { TEST_DO(test_dot_product(((2 * 7) + (3 * 11) + (5 * 13)), - spec(x(3), Seq({ 2, 3, 5 })), - spec(x(3), Seq({ 7, 11, 13 })))); + {x(3)}, Seq({ 2, 3, 5 }), + {x(3)}, Seq({ 7, 11, 13 }))); } //------------------------------------------------------------------------- @@ -714,6 +742,16 @@ struct TestContext { spec({x(2),y(2),z(3)}, Seq({1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0})))); TEST_DO(test_concat(spec(y(2), Seq({1.0, 2.0})), spec(y(2), Seq({4.0, 5.0})), "x", spec({x(2), y(2)}, Seq({1.0, 2.0, 4.0, 5.0})))); + + TEST_DO(test_concat(spec(float_cells({x(1)}), Seq({10.0})), spec(20.0), "x", spec(float_cells({x(2)}), Seq({10.0, 20.0})))); + TEST_DO(test_concat(spec(10.0), spec(float_cells({x(1)}), Seq({20.0})), "x", spec(float_cells({x(2)}), Seq({10.0, 20.0})))); + + TEST_DO(test_concat(spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})), spec(x(2), Seq({4.0, 5.0})), "x", + spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); + TEST_DO(test_concat(spec(x(3), Seq({1.0, 2.0, 3.0})), spec(float_cells({x(2)}), Seq({4.0, 5.0})), "x", + spec(x(5), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); + TEST_DO(test_concat(spec(float_cells({x(3)}), Seq({1.0, 2.0, 3.0})), spec(float_cells({x(2)}), Seq({4.0, 5.0})), "x", + spec(float_cells({x(5)}), Seq({1.0, 2.0, 3.0, 4.0, 5.0})))); } //------------------------------------------------------------------------- @@ -732,6 +770,7 @@ struct TestContext { void test_rename() { TEST_DO(test_rename("rename(a,x,y)", spec(x(5), N()), {"x"}, {"y"}, spec(y(5), N()))); TEST_DO(test_rename("rename(a,y,x)", spec({y(5),z(5)}, N()), {"y"}, {"x"}, spec({x(5),z(5)}, N()))); + TEST_DO(test_rename("rename(a,y,x)", spec(float_cells({y(5),z(5)}), N()), {"y"}, {"x"}, spec(float_cells({x(5),z(5)}), N()))); TEST_DO(test_rename("rename(a,z,x)", spec({y(5),z(5)}, N()), {"z"}, {"x"}, spec({y(5),x(5)}, N()))); TEST_DO(test_rename("rename(a,x,z)", spec({x(5),y(5)}, N()), {"x"}, {"z"}, spec({z(5),y(5)}, N()))); TEST_DO(test_rename("rename(a,y,z)", spec({x(5),y(5)}, N()), {"y"}, {"z"}, spec({x(5),z(5)}, N()))); @@ -746,6 +785,7 @@ struct TestContext { void test_tensor_lambda() { TEST_DO(test_tensor_lambda("tensor(x[10])(x+1)", spec(x(10), N()))); + TEST_DO(test_tensor_lambda("tensor<float>(x[10])(x+1)", spec(float_cells({x(10)}), N()))); TEST_DO(test_tensor_lambda("tensor(x[5],y[4])(x*4+(y+1))", spec({x(5),y(4)}, N()))); TEST_DO(test_tensor_lambda("tensor(x[5],y[4])(x==y)", spec({x(5),y(4)}, Seq({ 1.0, 0.0, 0.0, 0.0, @@ -818,11 +858,14 @@ struct TestContext { TEST_DO(verify_encode_decode(spec({x(3)}, N()))); TEST_DO(verify_encode_decode(spec({x(3),y(5)}, N()))); TEST_DO(verify_encode_decode(spec({x(3),y(5),z(7)}, N()))); + TEST_DO(verify_encode_decode(spec(float_cells({x(3),y(5),z(7)}), N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"})}, N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y({"foo","bar"})}, N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, N()))); + TEST_DO(verify_encode_decode(spec(float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), N()))); TEST_DO(verify_encode_decode(spec({x(3),y({"foo", "bar"}),z(7)}, N()))); TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y(5),z({"i","j","k","l"})}, N()))); + TEST_DO(verify_encode_decode(spec(float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}), N()))); } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/test/tensor_model.hpp b/eval/src/vespa/eval/eval/test/tensor_model.hpp index 50a7b6a639a..4fad2820cf7 100644 --- a/eval/src/vespa/eval/eval/test/tensor_model.hpp +++ b/eval/src/vespa/eval/eval/test/tensor_model.hpp @@ -10,6 +10,7 @@ namespace vespalib { namespace eval { namespace test { +using CellType = ValueType::CellType; using map_fun_t = TensorEngine::map_fun_t; using join_fun_t = TensorEngine::join_fun_t; @@ -146,7 +147,22 @@ struct Domain { Domain::Domain(const Domain &) = default; Domain::~Domain() {} -using Layout = std::vector<Domain>; +struct Layout { + CellType cell_type; + std::vector<Domain> domains; + Layout(std::initializer_list<Domain> domains_in) + : cell_type(CellType::DOUBLE), domains(domains_in) {} + Layout(CellType cell_type_in, std::vector<Domain> domains_in) + : cell_type(cell_type_in), domains(std::move(domains_in)) {} + auto begin() const { return domains.begin(); } + auto end() const { return domains.end(); } + auto size() const { return domains.size(); } + auto operator[](size_t idx) const { return domains[idx]; } +}; + +Layout float_cells(const Layout &layout) { + return Layout(CellType::FLOAT, layout.domains); +} Domain x() { return Domain("x", {}); } Domain x(size_t size) { return Domain("x", size); } @@ -162,9 +178,6 @@ Domain z(const std::vector<vespalib::string> &keys) { return Domain("z", keys); // Infer the tensor type spanned by the given spaces vespalib::string infer_type(const Layout &layout) { - if (layout.empty()) { - return "double"; - } std::vector<ValueType::Dimension> dimensions; for (const auto &domain: layout) { if (domain.size == 0) { @@ -173,7 +186,7 @@ vespalib::string infer_type(const Layout &layout) { dimensions.emplace_back(domain.dimension, domain.size); // indexed } } - return ValueType::tensor_type(dimensions).to_spec(); + return ValueType::tensor_type(dimensions, layout.cell_type).to_spec(); } // Wrapper for the things needed to generate a tensor diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b969e328419..5e9c0e2543c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -151,6 +151,12 @@ public class Flags { "Takes effect on deployment through controller", APPLICATION_ID); + public static final UnboundBooleanFlag MULTIPLE_GLOBAL_ENDPOINTS = defineFeatureFlag( + "multiple-global-endpoints", false, + "Allow applications to use new endpoints syntax in deployment.xml", + "Takes effect on deployment through controller", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index d7f41c4d8e2..438732ad4a8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -41,8 +40,7 @@ import java.util.stream.Collectors; * to failed due to some undetected hardware failure will end up being failed again. * When that has happened enough they will not be recycled. * <p> - * The Chef recipe running locally on the node may set hardwareFailureDescription to avoid the node - * being automatically recycled in cases where an error has been positively detected. + * Nodes with detected hardware issues will not be recycled. * * @author bratseth * @author mpolden diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index eea6db8c782..6dbe6b1c2a5 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -9,6 +9,7 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vespalib/util/assert.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include "distributor_bucket_space_repo.h" #include "distributor_bucket_space.h" @@ -151,12 +152,13 @@ void IdealStateManager::verify_only_live_nodes_in_context(const StateChecker::Co const auto& state = c.systemState.getNodeState(lib::Node(lib::NodeType::STORAGE, index)); // Only nodes in Up, Initializing or Retired should ever be present in the DB. if (!state.getState().oneOf("uir")) { - LOG(warning, "%s in bucket DB is on node %u, which is in unavailable state %s. " - "Current cluster state is '%s'", - c.entry.getBucketId().toString().c_str(), - index, - state.getState().toString().c_str(), - c.systemState.toString().c_str()); + LOG(error, "%s in bucket DB is on node %u, which is in unavailable state %s. " + "Current cluster state is '%s'", + c.entry.getBucketId().toString().c_str(), + index, + state.getState().toString().c_str(), + c.systemState.toString().c_str()); + ASSERT_ONCE_OR_LOG(false, "Bucket DB contains replicas on unavailable node", 10000); _has_logged_phantom_replica_warning = true; } } |